Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
home:ymarkovitch
openssl-1_1
openssl-1_1-Optimize-ppc64.patch
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File openssl-1_1-Optimize-ppc64.patch of Package openssl-1_1
From 4dba53694bf633c272075e62acdc5a5ca3003ce6 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs <amitay@ozlabs.org> Date: Mon, 29 Mar 2021 18:06:13 +1100 Subject: [PATCH 01/29] numbers: Define 128-bit integers if compiler supports Signed-off-by: Amitay Isaacs <amitay@ozlabs.org> Reviewed-by: Tomas Mraz <tomas@openssl.org> Reviewed-by: Matt Caswell <matt@openssl.org> (Merged from https://github.com/openssl/openssl/pull/14784) (cherry picked from commit bbed0d1cbd436af6797d7837e270bff4ca4d5a10) --- include/internal/numbers.h | 10 ++++++++++ 1 file changed, 10 insertions(+) Index: openssl-1.1.1l/include/internal/numbers.h =================================================================== --- openssl-1.1.1l.orig/include/internal/numbers.h +++ openssl-1.1.1l/include/internal/numbers.h @@ -60,6 +60,16 @@ # define UINT64_MAX __MAXUINT__(uint64_t) # endif +# ifndef INT128_MAX +# if defined(__SIZEOF_INT128__) && __SIZEOF_INT128__ == 16 +typedef __int128_t int128_t; +typedef __uint128_t uint128_t; +# define INT128_MIN __MININT__(int128_t) +# define INT128_MAX __MAXINT__(int128_t) +# define UINT128_MAX __MAXUINT__(uint128_t) +# endif +# endif + # ifndef SIZE_MAX # define SIZE_MAX __MAXUINT__(size_t) # endif Index: openssl-1.1.1l/crypto/bn/bn_div.c =================================================================== --- openssl-1.1.1l.orig/crypto/bn/bn_div.c +++ openssl-1.1.1l/crypto/bn/bn_div.c @@ -97,7 +97,7 @@ BN_ULONG bn_div_3_words(const BN_ULONG * */ # if BN_BITS2 == 64 && defined(__SIZEOF_INT128__) && __SIZEOF_INT128__==16 # undef BN_ULLONG -# define BN_ULLONG __uint128_t +# define BN_ULLONG uint128_t # define BN_LLONG # endif Index: openssl-1.1.1l/crypto/bn/bn_local.h =================================================================== --- openssl-1.1.1l.orig/crypto/bn/bn_local.h +++ openssl-1.1.1l/crypto/bn/bn_local.h @@ -22,6 +22,7 @@ # endif # include "crypto/bn.h" +# include "internal/numbers.h" /* * These preprocessor symbols control various aspects of the bignum headers @@ -374,9 +375,9 @@ struct bn_gencb_st { */ # if defined(__SIZEOF_INT128__) && __SIZEOF_INT128__==16 && \ (defined(SIXTY_FOUR_BIT) || defined(SIXTY_FOUR_BIT_LONG)) -# define BN_UMULT_HIGH(a,b) (((__uint128_t)(a)*(b))>>64) +# define BN_UMULT_HIGH(a,b) (((uint128_t)(a)*(b))>>64) # define BN_UMULT_LOHI(low,high,a,b) ({ \ - __uint128_t ret=(__uint128_t)(a)*(b); \ + uint128_t ret=(uint128_t)(a)*(b); \ (high)=ret>>64; (low)=ret; }) # elif defined(__alpha) && (defined(SIXTY_FOUR_BIT_LONG) || defined(SIXTY_FOUR_BIT)) # if defined(__DECC) Index: openssl-1.1.1l/crypto/ec/curve25519.c =================================================================== --- openssl-1.1.1l.orig/crypto/ec/curve25519.c +++ openssl-1.1.1l/crypto/ec/curve25519.c @@ -11,6 +11,8 @@ #include "ec_local.h" #include <openssl/sha.h> +#include "internal/numbers.h" + #if defined(X25519_ASM) && (defined(__x86_64) || defined(__x86_64__) || \ defined(_M_AMD64) || defined(_M_X64)) @@ -252,7 +254,7 @@ static void x25519_scalar_mulx(uint8_t o #endif #if defined(X25519_ASM) \ - || ( (defined(__SIZEOF_INT128__) && __SIZEOF_INT128__ == 16) \ + || ( defined(INT128_MAX) \ && !defined(__sparc__) \ && (!defined(__SIZEOF_LONG__) || (__SIZEOF_LONG__ == 8)) \ && !(defined(__ANDROID__) && !defined(__clang__)) ) @@ -385,7 +387,7 @@ void x25519_fe51_mul121666(fe51 h, fe51 # define fe51_mul121666 x25519_fe51_mul121666 # else -typedef __uint128_t u128; +typedef uint128_t u128; static void fe51_mul(fe51 h, const fe51 f, const fe51 g) { Index: openssl-1.1.1l/crypto/ec/curve448/curve448utils.h =================================================================== --- openssl-1.1.1l.orig/crypto/ec/curve448/curve448utils.h +++ openssl-1.1.1l/crypto/ec/curve448/curve448utils.h @@ -15,6 +15,8 @@ # include <openssl/e_os2.h> +# include "internal/numbers.h" + /* * Internal word types. Somewhat tricky. This could be decided separately per * platform. However, the structs do need to be all the same size and @@ -41,9 +43,9 @@ typedef int64_t c448_sword_t; /* "Boolean" type, will be set to all-zero or all-one (i.e. -1u) */ typedef uint64_t c448_bool_t; /* Double-word size for internal computations */ -typedef __uint128_t c448_dword_t; +typedef uint128_t c448_dword_t; /* Signed double-word size for internal computations */ -typedef __int128_t c448_dsword_t; +typedef int128_t c448_dsword_t; # elif C448_WORD_BITS == 32 /* Word size for internal computations */ typedef uint32_t c448_word_t; Index: openssl-1.1.1l/crypto/ec/curve448/word.h =================================================================== --- openssl-1.1.1l.orig/crypto/ec/curve448/word.h +++ openssl-1.1.1l/crypto/ec/curve448/word.h @@ -17,15 +17,20 @@ # include <assert.h> # include <stdlib.h> # include <openssl/e_os2.h> -# include "arch_intrinsics.h" # include "curve448utils.h" +# ifdef INT128_MAX +# include "arch_64/arch_intrinsics.h" +# else +# include "arch_32/arch_intrinsics.h" +# endif + # if (ARCH_WORD_BITS == 64) typedef uint64_t word_t, mask_t; -typedef __uint128_t dword_t; +typedef uint128_t dword_t; typedef int32_t hsword_t; typedef int64_t sword_t; -typedef __int128_t dsword_t; +typedef int128_t dsword_t; # elif (ARCH_WORD_BITS == 32) typedef uint32_t word_t, mask_t; typedef uint64_t dword_t; Index: openssl-1.1.1l/crypto/ec/ecp_nistp224.c =================================================================== --- openssl-1.1.1l.orig/crypto/ec/ecp_nistp224.c +++ openssl-1.1.1l/crypto/ec/ecp_nistp224.c @@ -40,11 +40,9 @@ NON_EMPTY_TRANSLATION_UNIT # include <openssl/err.h> # include "ec_local.h" -# if defined(__SIZEOF_INT128__) && __SIZEOF_INT128__==16 - /* even with gcc, the typedef won't work for 32-bit platforms */ -typedef __uint128_t uint128_t; /* nonstandard; implemented by gcc on 64-bit - * platforms */ -# else +#include "internal/numbers.h" + +#ifndef INT128_MAX # error "Your compiler doesn't appear to support 128-bit integer types" # endif Index: openssl-1.1.1l/crypto/ec/ecp_nistp256.c =================================================================== --- openssl-1.1.1l.orig/crypto/ec/ecp_nistp256.c +++ openssl-1.1.1l/crypto/ec/ecp_nistp256.c @@ -41,14 +41,11 @@ NON_EMPTY_TRANSLATION_UNIT # include <openssl/err.h> # include "ec_local.h" -# if defined(__SIZEOF_INT128__) && __SIZEOF_INT128__==16 - /* even with gcc, the typedef won't work for 32-bit platforms */ -typedef __uint128_t uint128_t; /* nonstandard; implemented by gcc on 64-bit - * platforms */ -typedef __int128_t int128_t; -# else -# error "Your compiler doesn't appear to support 128-bit integer types" -# endif +#include "internal/numbers.h" + +#ifndef INT128_MAX +# error "Your compiler doesn't appear to support 128-bit integer types" +#endif typedef uint8_t u8; typedef uint32_t u32; Index: openssl-1.1.1l/crypto/ec/ecp_nistp521.c =================================================================== --- openssl-1.1.1l.orig/crypto/ec/ecp_nistp521.c +++ openssl-1.1.1l/crypto/ec/ecp_nistp521.c @@ -40,13 +40,11 @@ NON_EMPTY_TRANSLATION_UNIT # include <openssl/err.h> # include "ec_local.h" -# if defined(__SIZEOF_INT128__) && __SIZEOF_INT128__==16 - /* even with gcc, the typedef won't work for 32-bit platforms */ -typedef __uint128_t uint128_t; /* nonstandard; implemented by gcc on 64-bit - * platforms */ -# else -# error "Your compiler doesn't appear to support 128-bit integer types" -# endif +#include "internal/numbers.h" + +#ifndef INT128_MAX +# error "Your compiler doesn't appear to support 128-bit integer types" +#endif typedef uint8_t u8; typedef uint64_t u64; @@ -400,7 +398,7 @@ static void felem_diff128(largefelem out * On exit: * out[i] < 17 * max(in[i]) * max(in[i]) */ -static void felem_square(largefelem out, const felem in) +static void felem_square_ref(largefelem out, const felem in) { felem inx2, inx4; felem_scalar(inx2, in, 2); @@ -484,7 +482,7 @@ static void felem_square(largefelem out, * On exit: * out[i] < 17 * max(in1[i]) * max(in2[i]) */ -static void felem_mul(largefelem out, const felem in1, const felem in2) +static void felem_mul_ref(largefelem out, const felem in1, const felem in2) { felem in2x2; felem_scalar(in2x2, in2, 2); @@ -674,6 +672,57 @@ static void felem_reduce(felem out, cons */ } +#if defined(ECP_NISTP521_ASM) +void felem_square_wrapper(largefelem out, const felem in); +void felem_mul_wrapper(largefelem out, const felem in1, const felem in2); + +static void (*felem_square_p)(largefelem out, const felem in) = + felem_square_wrapper; +static void (*felem_mul_p)(largefelem out, const felem in1, const felem in2) = + felem_mul_wrapper; + +void p521_felem_square(largefelem out, const felem in); +void p521_felem_mul(largefelem out, const felem in1, const felem in2); + +# if defined(_ARCH_PPC64) +# include "../ppc_arch.h" +# endif + +void felem_select(void) +{ +# if defined(_ARCH_PPC64) + if ((OPENSSL_ppccap_P & PPC_MADD300) && (OPENSSL_ppccap_P & PPC_ALTIVEC)) { + felem_square_p = p521_felem_square; + felem_mul_p = p521_felem_mul; + + return; + } +# endif + + /* Default */ + felem_square_p = felem_square_ref; + felem_mul_p = felem_mul_ref; +} + +void felem_square_wrapper(largefelem out, const felem in) +{ + felem_select(); + felem_square_p(out, in); +} + +void felem_mul_wrapper(largefelem out, const felem in1, const felem in2) +{ + felem_select(); + felem_mul_p(out, in1, in2); +} + +# define felem_square felem_square_p +# define felem_mul felem_mul_p +#else +# define felem_square felem_square_ref +# define felem_mul felem_mul_ref +#endif + static void felem_square_reduce(felem out, const felem in) { largefelem tmp; Index: openssl-1.1.1l/crypto/poly1305/poly1305.c =================================================================== --- openssl-1.1.1l.orig/crypto/poly1305/poly1305.c +++ openssl-1.1.1l/crypto/poly1305/poly1305.c @@ -95,11 +95,10 @@ poly1305_blocks(void *ctx, const unsigne (a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1) \ ) -# if (defined(__SIZEOF_INT128__) && __SIZEOF_INT128__==16) && \ - (defined(__SIZEOF_LONG__) && __SIZEOF_LONG__==8) +# if defined(INT64_MAX) && defined(INT128_MAX) typedef unsigned long u64; -typedef __uint128_t u128; +typedef uint128_t u128; typedef struct { u64 h[3]; Index: openssl-1.1.1l/crypto/poly1305/poly1305_base2_44.c =================================================================== --- openssl-1.1.1l.orig/crypto/poly1305/poly1305_base2_44.c +++ openssl-1.1.1l/crypto/poly1305/poly1305_base2_44.c @@ -18,7 +18,7 @@ typedef unsigned char u8; typedef unsigned int u32; typedef unsigned long u64; -typedef unsigned __int128 u128; +typedef uint128_t u128; typedef struct { u64 h[3]; Index: openssl-1.1.1l/crypto/ec/build.info =================================================================== --- openssl-1.1.1l.orig/crypto/ec/build.info +++ openssl-1.1.1l/crypto/ec/build.info @@ -6,8 +13,9 @@ SOURCE[../../libcrypto]=\ ecp_nistp224.c ecp_nistp256.c ecp_nistp521.c ecp_nistputil.c \ ecp_oct.c ec2_oct.c ec_oct.c ec_kmeth.c ecdh_ossl.c ecdh_kdf.c \ ecdsa_ossl.c ecdsa_sign.c ecdsa_vrf.c curve25519.c ecx_meth.c \ - curve448/arch_32/f_impl.c curve448/f_generic.c curve448/scalar.c \ + curve448/f_generic.c curve448/scalar.c \ curve448/curve448_tables.c curve448/eddsa.c curve448/curve448.c \ + curve448/arch_64/f_impl64.c curve448/arch_32/f_impl32.c \ {- $target{ec_asm_src} -} GENERATE[ecp_nistz256-x86.s]=asm/ecp_nistz256-x86.pl \ @@ -29,6 +38,8 @@ GENERATE[ecp_nistz256-armv8.S]=asm/ecp_n INCLUDE[ecp_nistz256-armv8.o]=.. GENERATE[ecp_nistz256-ppc64.s]=asm/ecp_nistz256-ppc64.pl $(PERLASM_SCHEME) +GENERATE[ecp_nistp521-ppc64.s]=asm/ecp_nistp521-ppc64.pl $(PERLASM_SCHEME) + GENERATE[x25519-x86_64.s]=asm/x25519-x86_64.pl $(PERLASM_SCHEME) GENERATE[x25519-ppc64.s]=asm/x25519-ppc64.pl $(PERLASM_SCHEME) @@ -36,10 +47,3 @@ BEGINRAW[Makefile] {- $builddir -}/ecp_nistz256-%.S: {- $sourcedir -}/asm/ecp_nistz256-%.pl CC="$(CC)" $(PERL) $< $(PERLASM_SCHEME) $@ ENDRAW[Makefile] - -INCLUDE[curve448/arch_32/f_impl.o]=curve448/arch_32 curve448 -INCLUDE[curve448/f_generic.o]=curve448/arch_32 curve448 -INCLUDE[curve448/scalar.o]=curve448/arch_32 curve448 -INCLUDE[curve448/curve448_tables.o]=curve448/arch_32 curve448 -INCLUDE[curve448/eddsa.o]=curve448/arch_32 curve448 -INCLUDE[curve448/curve448.o]=curve448/arch_32 curve448 Index: openssl-1.1.1l/crypto/ec/curve448/field.h =================================================================== --- openssl-1.1.1l.orig/crypto/ec/curve448/field.h +++ openssl-1.1.1l/crypto/ec/curve448/field.h @@ -66,10 +66,15 @@ void gf_serialize(uint8_t *serial, const mask_t gf_deserialize(gf x, const uint8_t serial[SER_BYTES], int with_hibit, uint8_t hi_nmask); -# include "f_impl.h" /* Bring in the inline implementations */ # define LIMBPERM(i) (i) -# define LIMB_MASK(i) (((1)<<LIMB_PLACE_VALUE(i))-1) +# if (ARCH_WORD_BITS == 32) +# include "arch_32/f_impl.h" /* Bring in the inline implementations */ +# define LIMB_MASK(i) (((1)<<LIMB_PLACE_VALUE(i))-1) +# elif (ARCH_WORD_BITS == 64) +# include "arch_64/f_impl.h" /* Bring in the inline implementations */ +# define LIMB_MASK(i) (((1ULL)<<LIMB_PLACE_VALUE(i))-1) +# endif static const gf ZERO = {{{0}}}, ONE = {{{1}}}; Index: openssl-1.1.1l/crypto/ec/curve448/arch_64/arch_intrinsics.h =================================================================== --- /dev/null +++ openssl-1.1.1l/crypto/ec/curve448/arch_64/arch_intrinsics.h @@ -0,0 +1,27 @@ +/* + * Copyright 2017-2018 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016 Cryptography Research, Inc. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + * + * Originally written by Mike Hamburg + */ + +# include "internal/constant_time.h" + +#ifndef OSSL_CRYPTO_EC_CURVE448_ARCH_64_INTRINSICS_H +# define OSSL_CRYPTO_EC_CURVE448_ARCH_64_INTRINSICS_H + +# define ARCH_WORD_BITS 64 + +# define word_is_zero(a) constant_time_is_zero_64(a) + +static ossl_inline uint128_t widemul(uint64_t a, uint64_t b) +{ + return ((uint128_t) a) * b; +} + +#endif /* OSSL_CRYPTO_EC_CURVE448_ARCH_64_INTRINSICS_H */ Index: openssl-1.1.1l/crypto/ec/curve448/arch_64/f_impl.h =================================================================== --- /dev/null +++ openssl-1.1.1l/crypto/ec/curve448/arch_64/f_impl.h @@ -0,0 +1,58 @@ +/* + * Copyright 2017-2018 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2014-2016 Cryptography Research, Inc. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + * + * Originally written by Mike Hamburg + */ + +#ifndef OSSL_CRYPTO_EC_CURVE448_ARCH_64_F_IMPL_H +# define OSSL_CRYPTO_EC_CURVE448_ARCH_64_F_IMPL_H + +# define GF_HEADROOM 9999 /* Everything is reduced anyway */ +# define FIELD_LITERAL(a,b,c,d,e,f,g,h) {{a,b,c,d,e,f,g,h}} + +# define LIMB_PLACE_VALUE(i) 56 + +void gf_add_RAW(gf out, const gf a, const gf b) +{ + unsigned int i; + + for (i = 0; i < NLIMBS; i++) + out->limb[i] = a->limb[i] + b->limb[i]; + + gf_weak_reduce(out); +} + +void gf_sub_RAW(gf out, const gf a, const gf b) +{ + uint64_t co1 = ((1ULL << 56) - 1) * 2, co2 = co1 - 2; + unsigned int i; + + for (i = 0; i < NLIMBS; i++) + out->limb[i] = a->limb[i] - b->limb[i] + ((i == NLIMBS / 2) ? co2 : co1); + + gf_weak_reduce(out); +} + +void gf_bias(gf a, int amt) +{ +} + +void gf_weak_reduce(gf a) +{ + uint64_t mask = (1ULL << 56) - 1; + uint64_t tmp = a->limb[NLIMBS - 1] >> 56; + unsigned int i; + + a->limb[NLIMBS / 2] += tmp; + for (i = NLIMBS - 1; i > 0; i--) + a->limb[i] = (a->limb[i] & mask) + (a->limb[i - 1] >> 56); + a->limb[0] = (a->limb[0] & mask) + tmp; +} + +#endif /* OSSL_CRYPTO_EC_CURVE448_ARCH_64_F_IMPL_H */ Index: openssl-1.1.1l/include/internal/constant_time.h =================================================================== --- openssl-1.1.1l.orig/include/internal/constant_time.h +++ openssl-1.1.1l/include/internal/constant_time.h @@ -181,6 +181,11 @@ static ossl_inline uint32_t constant_tim return constant_time_msb_32(~a & (a - 1)); } +static ossl_inline uint64_t constant_time_is_zero_64(uint64_t a) +{ + return constant_time_msb_64(~a & (a - 1)); +} + static ossl_inline unsigned int constant_time_eq(unsigned int a, unsigned int b) { Index: openssl-1.1.1l/crypto/ec/curve448/arch_32/f_impl32.c =================================================================== --- /dev/null +++ openssl-1.1.1l/crypto/ec/curve448/arch_32/f_impl32.c @@ -0,0 +1,104 @@ +/* + * Copyright 2017-2018 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2014 Cryptography Research, Inc. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + * + * Originally written by Mike Hamburg + */ + +#include <openssl/opensslconf.h> +#include "internal/numbers.h" + +#ifdef UINT128_MAX +/* We have support for 128 bit ints, so do nothing here */ +NON_EMPTY_TRANSLATION_UNIT +#else + +# include "../field.h" + +void gf_mul(gf_s * RESTRICT cs, const gf as, const gf bs) +{ + const uint32_t *a = as->limb, *b = bs->limb; + uint32_t *c = cs->limb; + uint64_t accum0 = 0, accum1 = 0, accum2 = 0; + uint32_t mask = (1 << 28) - 1; + uint32_t aa[8], bb[8]; + int i, j; + + for (i = 0; i < 8; i++) { + aa[i] = a[i] + a[i + 8]; + bb[i] = b[i] + b[i + 8]; + } + + for (j = 0; j < 8; j++) { + accum2 = 0; + for (i = 0; i < j + 1; i++) { + accum2 += widemul(a[j - i], b[i]); + accum1 += widemul(aa[j - i], bb[i]); + accum0 += widemul(a[8 + j - i], b[8 + i]); + } + accum1 -= accum2; + accum0 += accum2; + accum2 = 0; + for (i = j + 1; i < 8; i++) { + accum0 -= widemul(a[8 + j - i], b[i]); + accum2 += widemul(aa[8 + j - i], bb[i]); + accum1 += widemul(a[16 + j - i], b[8 + i]); + } + accum1 += accum2; + accum0 += accum2; + c[j] = ((uint32_t)(accum0)) & mask; + c[j + 8] = ((uint32_t)(accum1)) & mask; + accum0 >>= 28; + accum1 >>= 28; + } + + accum0 += accum1; + accum0 += c[8]; + accum1 += c[0]; + c[8] = ((uint32_t)(accum0)) & mask; + c[0] = ((uint32_t)(accum1)) & mask; + + accum0 >>= 28; + accum1 >>= 28; + c[9] += ((uint32_t)(accum0)); + c[1] += ((uint32_t)(accum1)); +} + +void gf_mulw_unsigned(gf_s * RESTRICT cs, const gf as, uint32_t b) +{ + const uint32_t *a = as->limb; + uint32_t *c = cs->limb; + uint64_t accum0 = 0, accum8 = 0; + uint32_t mask = (1 << 28) - 1; + int i; + + assert(b <= mask); + + for (i = 0; i < 8; i++) { + accum0 += widemul(b, a[i]); + accum8 += widemul(b, a[i + 8]); + c[i] = accum0 & mask; + accum0 >>= 28; + c[i + 8] = accum8 & mask; + accum8 >>= 28; + } + + accum0 += accum8 + c[8]; + c[8] = ((uint32_t)accum0) & mask; + c[9] += (uint32_t)(accum0 >> 28); + + accum8 += c[0]; + c[0] = ((uint32_t)accum8) & mask; + c[1] += (uint32_t)(accum8 >> 28); +} + +void gf_sqr(gf_s * RESTRICT cs, const gf as) +{ + gf_mul(cs, as, as); /* Performs better with a dedicated square */ +} +#endif Index: openssl-1.1.1l/crypto/ec/curve448/arch_64/f_impl64.c =================================================================== --- /dev/null +++ openssl-1.1.1l/crypto/ec/curve448/arch_64/f_impl64.c @@ -0,0 +1,210 @@ +/* + * Copyright 2017-2018 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2014 Cryptography Research, Inc. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + * + * Originally written by Mike Hamburg + */ + +#include <openssl/opensslconf.h> +#include "internal/numbers.h" + +#ifndef UINT128_MAX +/* No support for 128 bit ints, so do nothing here */ +NON_EMPTY_TRANSLATION_UNIT +#else + +# include "../field.h" + +void gf_mul(gf_s * RESTRICT cs, const gf as, const gf bs) +{ + const uint64_t *a = as->limb, *b = bs->limb; + uint64_t *c = cs->limb; + uint128_t accum0 = 0, accum1 = 0, accum2; + uint64_t mask = (1ULL << 56) - 1; + uint64_t aa[4], bb[4], bbb[4]; + unsigned int i, j; + + for (i = 0; i < 4; i++) { + aa[i] = a[i] + a[i + 4]; + bb[i] = b[i] + b[i + 4]; + bbb[i] = bb[i] + b[i + 4]; + } + + for (i = 0; i < 4; i++) { + accum2 = 0; + + for (j = 0; j <= i; j++) { + accum2 += widemul(a[j], b[i - j]); + accum1 += widemul(aa[j], bb[i - j]); + accum0 += widemul(a[j + 4], b[i - j + 4]); + } + for (; j < 4; j++) { + accum2 += widemul(a[j], b[i - j + 8]); + accum1 += widemul(aa[j], bbb[i - j + 4]); + accum0 += widemul(a[j + 4], bb[i - j + 4]); + } + + accum1 -= accum2; + accum0 += accum2; + + c[i] = ((uint64_t)(accum0)) & mask; + c[i + 4] = ((uint64_t)(accum1)) & mask; + + accum0 >>= 56; + accum1 >>= 56; + } + + accum0 += accum1; + accum0 += c[4]; + accum1 += c[0]; + c[4] = ((uint64_t)(accum0)) & mask; + c[0] = ((uint64_t)(accum1)) & mask; + + accum0 >>= 56; + accum1 >>= 56; + + c[5] += ((uint64_t)(accum0)); + c[1] += ((uint64_t)(accum1)); +} + +void gf_mulw_unsigned(gf_s * __restrict__ cs, const gf as, uint32_t b) +{ + const uint64_t *a = as->limb; + uint64_t *c = cs->limb; + uint128_t accum0 = 0, accum4 = 0; + uint64_t mask = (1ULL << 56) - 1; + int i; + + for (i = 0; i < 4; i++) { + accum0 += widemul(b, a[i]); + accum4 += widemul(b, a[i + 4]); + c[i] = accum0 & mask; + accum0 >>= 56; + c[i + 4] = accum4 & mask; + accum4 >>= 56; + } + + accum0 += accum4 + c[4]; + c[4] = accum0 & mask; + c[5] += accum0 >> 56; + + accum4 += c[0]; + c[0] = accum4 & mask; + c[1] += accum4 >> 56; +} + +void gf_sqr(gf_s * __restrict__ cs, const gf as) +{ + const uint64_t *a = as->limb; + uint64_t *c = cs->limb; + uint128_t accum0 = 0, accum1 = 0, accum2; + uint64_t mask = (1ULL << 56) - 1; + uint64_t aa[4]; + + /* For some reason clang doesn't vectorize this without prompting? */ + unsigned int i; + for (i = 0; i < 4; i++) { + aa[i] = a[i] + a[i + 4]; + } + + accum2 = widemul(a[0], a[3]); + accum0 = widemul(aa[0], aa[3]); + accum1 = widemul(a[4], a[7]); + + accum2 += widemul(a[1], a[2]); + accum0 += widemul(aa[1], aa[2]); + accum1 += widemul(a[5], a[6]); + + accum0 -= accum2; + accum1 += accum2; + + c[3] = ((uint64_t)(accum1)) << 1 & mask; + c[7] = ((uint64_t)(accum0)) << 1 & mask; + + accum0 >>= 55; + accum1 >>= 55; + + accum0 += widemul(2 * aa[1], aa[3]); + accum1 += widemul(2 * a[5], a[7]); + accum0 += widemul(aa[2], aa[2]); + accum1 += accum0; + + accum0 -= widemul(2 * a[1], a[3]); + accum1 += widemul(a[6], a[6]); + + accum2 = widemul(a[0], a[0]); + accum1 -= accum2; + accum0 += accum2; + + accum0 -= widemul(a[2], a[2]); + accum1 += widemul(aa[0], aa[0]); + accum0 += widemul(a[4], a[4]); + + c[0] = ((uint64_t)(accum0)) & mask; + c[4] = ((uint64_t)(accum1)) & mask; + + accum0 >>= 56; + accum1 >>= 56; + + accum2 = widemul(2 * aa[2], aa[3]); + accum0 -= widemul(2 * a[2], a[3]); + accum1 += widemul(2 * a[6], a[7]); + + accum1 += accum2; + accum0 += accum2; + + accum2 = widemul(2 * a[0], a[1]); + accum1 += widemul(2 * aa[0], aa[1]); + accum0 += widemul(2 * a[4], a[5]); + + accum1 -= accum2; + accum0 += accum2; + + c[1] = ((uint64_t)(accum0)) & mask; + c[5] = ((uint64_t)(accum1)) & mask; + + accum0 >>= 56; + accum1 >>= 56; + + accum2 = widemul(aa[3], aa[3]); + accum0 -= widemul(a[3], a[3]); + accum1 += widemul(a[7], a[7]); + + accum1 += accum2; + accum0 += accum2; + + accum2 = widemul(2 * a[0], a[2]); + accum1 += widemul(2 * aa[0], aa[2]); + accum0 += widemul(2 * a[4], a[6]); + + accum2 += widemul(a[1], a[1]); + accum1 += widemul(aa[1], aa[1]); + accum0 += widemul(a[5], a[5]); + + accum1 -= accum2; + accum0 += accum2; + + c[2] = ((uint64_t)(accum0)) & mask; + c[6] = ((uint64_t)(accum1)) & mask; + + accum0 >>= 56; + accum1 >>= 56; + + accum0 += c[3]; + accum1 += c[7]; + c[3] = ((uint64_t)(accum0)) & mask; + c[7] = ((uint64_t)(accum1)) & mask; + + /* we could almost stop here, but it wouldn't be stable, so... */ + + accum0 >>= 56; + accum1 >>= 56; + c[4] += ((uint64_t)(accum0)) + ((uint64_t)(accum1)); + c[0] += ((uint64_t)(accum1)); +} +#endif Index: openssl-1.1.1l/Configurations/00-base-templates.conf =================================================================== --- openssl-1.1.1l.orig/Configurations/00-base-templates.conf +++ openssl-1.1.1l/Configurations/00-base-templates.conf @@ -351,7 +351,8 @@ my %targets=( ppc64_asm => { inherit_from => [ "ppc32_asm" ], template => 1, - ec_asm_src => "ecp_nistz256.c ecp_nistz256-ppc64.s x25519-ppc64.s", + bn_asm_src => add("ppc64-mont-fixed.s"), + ec_asm_src => "ecp_nistz256.c ecp_nistz256-ppc64.s ecp_nistp521-ppc64.s x25519-ppc64.s", keccak1600_asm_src => "keccak1600-ppc64.s", }, ); Index: openssl-1.1.1l/crypto/bn/asm/ppc64-mont-fixed.pl =================================================================== --- /dev/null +++ openssl-1.1.1l/crypto/bn/asm/ppc64-mont-fixed.pl @@ -0,0 +1,581 @@ +#! /usr/bin/env perl +# Copyright 2021 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +# ==================================================================== +# Written by Amitay Isaacs <amitay@ozlabs.org>, Martin Schwenke +# <martin@meltin.net> & Alastair D'Silva <alastair@d-silva.org> for +# the OpenSSL project. +# ==================================================================== + +# +# Fixed length (n=6), unrolled PPC Montgomery Multiplication +# + +# 2021 +# +# Although this is a generic implementation for unrolling Montgomery +# Multiplication for arbitrary values of n, this is currently only +# used for n = 6 to improve the performance of ECC p384. +# +# Unrolling allows intermediate results to be stored in registers, +# rather than on the stack, improving performance by ~7% compared to +# the existing PPC assembly code. +# +# The ISA 3.0 implementation uses combination multiply/add +# instructions (maddld, maddhdu) to improve performance by an +# additional ~10% on Power 9. +# +# Finally, saving non-volatile registers into volatile vector +# registers instead of onto the stack saves a little more. +# +# On a Power 9 machine we see an overall improvement of ~18%. +# + +use strict; +use warnings; + +my ($flavour, $output, $dir, $xlate); + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or +die "can't locate ppc-xlate.pl"; + +open STDOUT,"| $^X $xlate $flavour \"$output\"" + or die "can't call $xlate: $!"; + +if ($flavour !~ /64/) { + die "bad flavour ($flavour) - only ppc64 permitted"; +} + +my $SIZE_T= 8; + +# Registers are global so the code is remotely readable + +# Parameters for Montgomery multiplication +my $sp = "r1"; +my $toc = "r2"; +my $rp = "r3"; +my $ap = "r4"; +my $bp = "r5"; +my $np = "r6"; +my $n0 = "r7"; +my $num = "r8"; + +my $i = "r9"; +my $c0 = "r10"; +my $bp0 = "r11"; +my $bpi = "r11"; +my $bpj = "r11"; +my $tj = "r12"; +my $apj = "r12"; +my $npj = "r12"; +my $lo = "r14"; +my $c1 = "r14"; + +# Non-volatile registers used for tp[i] +# +# 12 registers are available but the limit on unrolling is 10, +# since registers from $tp[0] to $tp[$n+1] are used. +my @tp = ("r20" .. "r31"); + +# volatile VSRs for saving non-volatile GPRs - faster than stack +my @vsrs = ("v32" .. "v46"); + +package Mont; + +sub new($$) +{ + my ($class, $n) = @_; + + if ($n > 10) { + die "Can't unroll for BN length ${n} (maximum 10)" + } + + my $self = { + code => "", + n => $n, + }; + bless $self, $class; + + return $self; +} + +sub add_code($$) +{ + my ($self, $c) = @_; + + $self->{code} .= $c; +} + +sub get_code($) +{ + my ($self) = @_; + + return $self->{code}; +} + +sub get_function_name($) +{ + my ($self) = @_; + + return "bn_mul_mont_fixed_n" . $self->{n}; +} + +sub get_label($$) +{ + my ($self, $l) = @_; + + return "L" . $l . "_" . $self->{n}; +} + +sub get_labels($@) +{ + my ($self, @labels) = @_; + + my %out = (); + + foreach my $l (@labels) { + $out{"$l"} = $self->get_label("$l"); + } + + return \%out; +} + +sub nl($) +{ + my ($self) = @_; + + $self->add_code("\n"); +} + +sub copy_result($) +{ + my ($self) = @_; + + my ($n) = $self->{n}; + + for (my $j = 0; $j < $n; $j++) { + $self->add_code(<<___); + std $tp[$j],`$j*$SIZE_T`($rp) +___ + } + +} + +sub mul_mont_fixed($) +{ + my ($self) = @_; + + my ($n) = $self->{n}; + my $fname = $self->get_function_name(); + my $label = $self->get_labels("outer", "enter", "sub", "copy", "end"); + + $self->add_code(<<___); + +.globl .${fname} +.align 5 +.${fname}: + +___ + + $self->save_registers(); + + $self->add_code(<<___); + ld $n0,0($n0) + + ld $bp0,0($bp) + + ld $apj,0($ap) +___ + + $self->mul_c_0($tp[0], $apj, $bp0, $c0); + + for (my $j = 1; $j < $n - 1; $j++) { + $self->add_code(<<___); + ld $apj,`$j*$SIZE_T`($ap) +___ + $self->mul($tp[$j], $apj, $bp0, $c0); + } + + $self->add_code(<<___); + ld $apj,`($n-1)*$SIZE_T`($ap) +___ + + $self->mul_last($tp[$n-1], $tp[$n], $apj, $bp0, $c0); + + $self->add_code(<<___); + li $tp[$n+1],0 + +___ + + $self->add_code(<<___); + li $i,0 + mtctr $num + b $label->{"enter"} + +.align 4 +$label->{"outer"}: + ldx $bpi,$bp,$i + + ld $apj,0($ap) +___ + + $self->mul_add_c_0($tp[0], $tp[0], $apj, $bpi, $c0); + + for (my $j = 1; $j < $n; $j++) { + $self->add_code(<<___); + ld $apj,`$j*$SIZE_T`($ap) +___ + $self->mul_add($tp[$j], $tp[$j], $apj, $bpi, $c0); + } + + $self->add_code(<<___); + addc $tp[$n],$tp[$n],$c0 + addze $tp[$n+1],$tp[$n+1] +___ + + $self->add_code(<<___); +.align 4 +$label->{"enter"}: + mulld $bpi,$tp[0],$n0 + + ld $npj,0($np) +___ + + $self->mul_add_c_0($lo, $tp[0], $bpi, $npj, $c0); + + for (my $j = 1; $j < $n; $j++) { + $self->add_code(<<___); + ld $npj,`$j*$SIZE_T`($np) +___ + $self->mul_add($tp[$j-1], $tp[$j], $npj, $bpi, $c0); + } + + $self->add_code(<<___); + addc $tp[$n-1],$tp[$n],$c0 + addze $tp[$n],$tp[$n+1] + + addi $i,$i,$SIZE_T + bdnz $label->{"outer"} + + and. $tp[$n],$tp[$n],$tp[$n] + bne $label->{"sub"} + + cmpld $tp[$n-1],$npj + blt $label->{"copy"} + +$label->{"sub"}: +___ + + # + # Reduction + # + + $self->add_code(<<___); + ld $bpj,`0*$SIZE_T`($np) + subfc $c1,$bpj,$tp[0] + std $c1,`0*$SIZE_T`($rp) + +___ + for (my $j = 1; $j < $n - 1; $j++) { + $self->add_code(<<___); + ld $bpj,`$j*$SIZE_T`($np) + subfe $c1,$bpj,$tp[$j] + std $c1,`$j*$SIZE_T`($rp) + +___ + } + + $self->add_code(<<___); + subfe $c1,$npj,$tp[$n-1] + std $c1,`($n-1)*$SIZE_T`($rp) + +___ + + $self->add_code(<<___); + addme. $tp[$n],$tp[$n] + beq $label->{"end"} + +$label->{"copy"}: +___ + + $self->copy_result(); + + $self->add_code(<<___); + +$label->{"end"}: +___ + + $self->restore_registers(); + + $self->add_code(<<___); + li r3,1 + blr +.size .${fname},.-.${fname} +___ + +} + +package Mont::GPR; + +our @ISA = ('Mont'); + +sub new($$) +{ + my ($class, $n) = @_; + + return $class->SUPER::new($n); +} + +sub save_registers($) +{ + my ($self) = @_; + + my $n = $self->{n}; + + $self->add_code(<<___); + std $lo,-8($sp) +___ + + for (my $j = 0; $j <= $n+1; $j++) { + $self->{code}.=<<___; + std $tp[$j],-`($j+2)*8`($sp) +___ + } + + $self->add_code(<<___); + +___ +} + +sub restore_registers($) +{ + my ($self) = @_; + + my $n = $self->{n}; + + $self->add_code(<<___); + ld $lo,-8($sp) +___ + + for (my $j = 0; $j <= $n+1; $j++) { + $self->{code}.=<<___; + ld $tp[$j],-`($j+2)*8`($sp) +___ + } + + $self->{code} .=<<___; + +___ +} + +# Direct translation of C mul() +sub mul($$$$$) +{ + my ($self, $r, $a, $w, $c) = @_; + + $self->add_code(<<___); + mulld $lo,$a,$w + addc $r,$lo,$c + mulhdu $c,$a,$w + addze $c,$c + +___ +} + +# Like mul() but $c is ignored as an input - an optimisation to save a +# preliminary instruction that would set input $c to 0 +sub mul_c_0($$$$$) +{ + my ($self, $r, $a, $w, $c) = @_; + + $self->add_code(<<___); + mulld $r,$a,$w + mulhdu $c,$a,$w + +___ +} + +# Like mul() but does not to the final addition of CA into $c - an +# optimisation to save an instruction +sub mul_last($$$$$$) +{ + my ($self, $r1, $r2, $a, $w, $c) = @_; + + $self->add_code(<<___); + mulld $lo,$a,$w + addc $r1,$lo,$c + mulhdu $c,$a,$w + + addze $r2,$c +___ +} + +# Like C mul_add() but allow $r_out and $r_in to be different +sub mul_add($$$$$$) +{ + my ($self, $r_out, $r_in, $a, $w, $c) = @_; + + $self->add_code(<<___); + mulld $lo,$a,$w + addc $lo,$lo,$c + mulhdu $c,$a,$w + addze $c,$c + addc $r_out,$r_in,$lo + addze $c,$c + +___ +} + +# Like mul_add() but $c is ignored as an input - an optimisation to save a +# preliminary instruction that would set input $c to 0 +sub mul_add_c_0($$$$$$) +{ + my ($self, $r_out, $r_in, $a, $w, $c) = @_; + + $self->add_code(<<___); + mulld $lo,$a,$w + addc $r_out,$r_in,$lo + mulhdu $c,$a,$w + addze $c,$c + +___ +} + +package Mont::GPR_300; + +our @ISA = ('Mont::GPR'); + +sub new($$) +{ + my ($class, $n) = @_; + + my $mont = $class->SUPER::new($n); + + return $mont; +} + +sub get_function_name($) +{ + my ($self) = @_; + + return "bn_mul_mont_300_fixed_n" . $self->{n}; +} + +sub get_label($$) +{ + my ($self, $l) = @_; + + return "L" . $l . "_300_" . $self->{n}; +} + +# Direct translation of C mul() +sub mul($$$$$) +{ + my ($self, $r, $a, $w, $c, $last) = @_; + + $self->add_code(<<___); + maddld $r,$a,$w,$c + maddhdu $c,$a,$w,$c + +___ +} + +# Save the last carry as the final entry +sub mul_last($$$$$) +{ + my ($self, $r1, $r2, $a, $w, $c) = @_; + + $self->add_code(<<___); + maddld $r1,$a,$w,$c + maddhdu $r2,$a,$w,$c + +___ +} + +# Like mul() but $c is ignored as an input - an optimisation to save a +# preliminary instruction that would set input $c to 0 +sub mul_c_0($$$$$) +{ + my ($self, $r, $a, $w, $c) = @_; + + $self->add_code(<<___); + mulld $r,$a,$w + mulhdu $c,$a,$w + +___ +} + +# Like C mul_add() but allow $r_out and $r_in to be different +sub mul_add($$$$$$) +{ + my ($self, $r_out, $r_in, $a, $w, $c) = @_; + + $self->add_code(<<___); + maddld $lo,$a,$w,$c + maddhdu $c,$a,$w,$c + addc $r_out,$r_in,$lo + addze $c,$c + +___ +} + +# Like mul_add() but $c is ignored as an input - an optimisation to save a +# preliminary instruction that would set input $c to 0 +sub mul_add_c_0($$$$$$) +{ + my ($self, $r_out, $r_in, $a, $w, $c) = @_; + + $self->add_code(<<___); + maddld $lo,$a,$w,$r_in + maddhdu $c,$a,$w,$r_in +___ + + if ($r_out ne $lo) { + $self->add_code(<<___); + mr $r_out,$lo +___ + } + + $self->nl(); +} + + +package main; + +my $code; + +$code.=<<___; +.machine "any" +.text +___ + +my $mont; + +$mont = new Mont::GPR(6); +$mont->mul_mont_fixed(); +$code .= $mont->get_code(); + +$mont = new Mont::GPR_300(6); +$mont->mul_mont_fixed(); +$code .= $mont->get_code(); + +$code =~ s/\`([^\`]*)\`/eval $1/gem; + +$code.=<<___; +.asciz "Montgomery Multiplication for PPC by <amitay\@ozlabs.org>, <alastair\@d-silva.org>" +___ + +print $code; +close STDOUT or die "error closing STDOUT: $!"; Index: openssl-1.1.1l/crypto/bn/build.info =================================================================== --- openssl-1.1.1l.orig/crypto/bn/build.info +++ openssl-1.1.1l/crypto/bn/build.info @@ -56,6 +56,7 @@ GENERATE[parisc-mont.s]=asm/parisc-mont. GENERATE[bn-ppc.s]=asm/ppc.pl $(PERLASM_SCHEME) GENERATE[ppc-mont.s]=asm/ppc-mont.pl $(PERLASM_SCHEME) GENERATE[ppc64-mont.s]=asm/ppc64-mont.pl $(PERLASM_SCHEME) +GENERATE[ppc64-mont-fixed.s]=asm/ppc64-mont-fixed.pl $(PERLASM_SCHEME) GENERATE[alpha-mont.S]=asm/alpha-mont.pl $(PERLASM_SCHEME) Index: openssl-1.1.1l/crypto/ppccap.c =================================================================== --- openssl-1.1.1l.orig/crypto/ppccap.c +++ openssl-1.1.1l/crypto/ppccap.c @@ -46,6 +46,12 @@ int bn_mul_mont(BN_ULONG *rp, const BN_U const BN_ULONG *np, const BN_ULONG *n0, int num); int bn_mul4x_mont_int(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, const BN_ULONG *np, const BN_ULONG *n0, int num); + int bn_mul_mont_fixed_n6(BN_ULONG *rp, const BN_ULONG *ap, + const BN_ULONG *bp, const BN_ULONG *np, + const BN_ULONG *n0, int num); + int bn_mul_mont_300_fixed_n6(BN_ULONG *rp, const BN_ULONG *ap, + const BN_ULONG *bp, const BN_ULONG *np, + const BN_ULONG *n0, int num); if (num < 4) return 0; @@ -61,6 +67,15 @@ int bn_mul_mont(BN_ULONG *rp, const BN_U * no opportunity to figure it out... */ +#if defined(_ARCH_PPC64) + if (num == 6) { + if (OPENSSL_ppccap_P & PPC_MADD300) + return bn_mul_mont_300_fixed_n6(rp, ap, bp, np, n0, num); + else + return bn_mul_mont_fixed_n6(rp, ap, bp, np, n0, num); + } +#endif + return bn_mul_mont_int(rp, ap, bp, np, n0, num); } #endif Index: openssl-1.1.1l/crypto/perlasm/ppc-xlate.pl =================================================================== --- openssl-1.1.1l.orig/crypto/perlasm/ppc-xlate.pl +++ openssl-1.1.1l/crypto/perlasm/ppc-xlate.pl @@ -136,6 +136,71 @@ my $quad = sub { }; ################################################################ +# vector register number hacking +################################################################ + +# It is convenient to be able to set a variable like: +# my $foo = "v33"; +# and use this in different contexts where: +# * a VSR (Vector-Scaler Register) number (i.e. "v33") is required +# * a VR (Vector Register) number (i.e. "v1") is required +# Map VSR numbering to VR number for certain vector instructions. + +# vs<N> -> v<N-32> if N > 32 +sub vsr2vr1 { + my $in = shift; + + my $n = int($in); + if ($n >= 32) { + $n -= 32; + } + + return "$n"; +} +# As above for first $num register args, returns list +sub _vsr2vr { + my $num = shift; + my @rest = @_; + my @subst = splice(@rest, 0, $num); + + @subst = map { vsr2vr1($_); } @subst; + + return (@subst, @rest); +} +# As above but 1st arg ($f) is extracted and reinserted after +# processing so that it can be ignored by a code generation function +# that consumes the result +sub vsr2vr_args { + my $num = shift; + my $f = shift; + + my @out = _vsr2vr($num, @_); + + return ($f, @out); +} +# As above but 1st arg is mnemonic, return formatted instruction +sub vsr2vr { + my $mnemonic = shift; + my $num = shift; + my $f = shift; + + my @out = _vsr2vr($num, @_); + + " ${mnemonic}${f} " . join(",", @out); +} + +# ISA 2.03 +my $vsel = sub { vsr2vr("vsel", 4, @_); }; +my $vsl = sub { vsr2vr("vsl", 3, @_); }; +my $vspltisb = sub { vsr2vr("vspltisb", 1, @_); }; +my $vspltisw = sub { vsr2vr("vspltisw", 1, @_); }; +my $vsr = sub { vsr2vr("vsr", 3, @_); }; +my $vsro = sub { vsr2vr("vsro", 3, @_); }; + +# ISA 3.0 +my $lxsd = sub { vsr2vr("lxsd", 1, @_); }; + +################################################################ # simplified mnemonics not handled by at least one assembler ################################################################ my $cmplw = sub { @@ -226,13 +291,18 @@ my $vpermdi = sub { # xxpermdi # PowerISA 2.07 stuff sub vcrypto_op { - my ($f, $vrt, $vra, $vrb, $op) = @_; + my ($f, $vrt, $vra, $vrb, $op) = vsr2vr_args(3, @_); " .long ".sprintf "0x%X",(4<<26)|($vrt<<21)|($vra<<16)|($vrb<<11)|$op; } sub vfour { my ($f, $vrt, $vra, $vrb, $vrc, $op) = @_; " .long ".sprintf "0x%X",(4<<26)|($vrt<<21)|($vra<<16)|($vrb<<11)|($vrc<<6)|$op; }; +sub vfour_vsr { + my ($f, $vrt, $vra, $vrb, $vrc, $op) = vsr2vr_args(4, @_); + " .long ".sprintf "0x%X",(4<<26)|($vrt<<21)|($vra<<16)|($vrb<<11)|($vrc<<6)|$op; +}; + my $vcipher = sub { vcrypto_op(@_, 1288); }; my $vcipherlast = sub { vcrypto_op(@_, 1289); }; my $vncipher = sub { vcrypto_op(@_, 1352); }; @@ -254,10 +324,10 @@ my $vsld = sub { vcrypto_op(@_, 1476); } my $vsrd = sub { vcrypto_op(@_, 1732); }; my $vsubudm = sub { vcrypto_op(@_, 1216); }; my $vaddcuq = sub { vcrypto_op(@_, 320); }; -my $vaddeuqm = sub { vfour(@_,60); }; -my $vaddecuq = sub { vfour(@_,61); }; -my $vmrgew = sub { vfour(@_,0,1932); }; -my $vmrgow = sub { vfour(@_,0,1676); }; +my $vaddeuqm = sub { vfour_vsr(@_,60); }; +my $vaddecuq = sub { vfour_vsr(@_,61); }; +my $vmrgew = sub { vfour_vsr(@_,0,1932); }; +my $vmrgow = sub { vfour_vsr(@_,0,1676); }; my $mtsle = sub { my ($f, $arg) = @_; @@ -298,7 +368,7 @@ my $addex = sub { my ($f, $rt, $ra, $rb, $cy) = @_; # only cy==0 is specified in 3.0B " .long ".sprintf "0x%X",(31<<26)|($rt<<21)|($ra<<16)|($rb<<11)|($cy<<9)|(170<<1); }; -my $vmsumudm = sub { vfour(@_,35); }; +my $vmsumudm = sub { vfour_vsr(@_, 35); }; while($line=<>) { Index: openssl-1.1.1l/Configurations/10-main.conf =================================================================== --- openssl-1.1.1l.orig/Configurations/10-main.conf +++ openssl-1.1.1l/Configurations/10-main.conf @@ -669,7 +669,7 @@ my %targets = ( inherit_from => [ "linux-generic64", asm("ppc64_asm") ], cflags => add("-m64"), cxxflags => add("-m64"), - lib_cppflags => add("-DB_ENDIAN"), + lib_cppflags => add("-DB_ENDIAN -DECP_NISTP521_ASM"), perlasm_scheme => "linux64", multilib => "64", }, @@ -677,7 +677,7 @@ my %targets = ( inherit_from => [ "linux-generic64", asm("ppc64_asm") ], cflags => add("-m64"), cxxflags => add("-m64"), - lib_cppflags => add("-DL_ENDIAN"), + lib_cppflags => add("-DL_ENDIAN -DECP_NISTP521_ASM"), perlasm_scheme => "linux64le", }, Index: openssl-1.1.1l/crypto/ec/asm/ecp_nistp521-ppc64.pl =================================================================== --- /dev/null +++ openssl-1.1.1l/crypto/ec/asm/ecp_nistp521-ppc64.pl @@ -0,0 +1,435 @@ +#! /usr/bin/env perl +# Copyright 2021 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the OpenSSL license (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# ==================================================================== +# Written by Amitay Isaacs <amitay@ozlabs.org> and Martin Schwenke +# <martin@meltin.net> for the OpenSSL project. +# ==================================================================== +# +# p521 lower-level primitives for PPC64 using vector instructions. +# + +use strict; +use warnings; + +my $flavour = shift; +my $output = ""; +while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} +if (!$output) { + $output = "-"; +} + +my ($xlate, $dir); +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or +die "can't locate ppc-xlate.pl"; + +open OUT,"| \"$^X\" $xlate $flavour $output"; +*STDOUT=*OUT; + +my $code = ""; + +my ($sp, $outp, $savelr, $savesp) = ("r1", "r3", "r10", "r12"); + +my $vzero = "v32"; + +sub startproc($) +{ + my ($name) = @_; + + $code.=<<___; + .globl ${name} + .align 5 +${name}: + +___ +} + +sub endproc($) +{ + my ($name) = @_; + + $code.=<<___; + blr + .size ${name},.-${name} + +___ +} + + +sub push_vrs($$) +{ + my ($min, $max) = @_; + + my $count = $max - $min + 1; + + $code.=<<___; + mr $savesp,$sp + stdu $sp,-16*`$count+1`($sp) + +___ + for (my $i = $min; $i <= $max; $i++) { + my $mult = $max - $i + 1; + $code.=<<___; + stxv $i,-16*$mult($savesp) +___ + + } + + $code.=<<___; + +___ +} + +sub pop_vrs($$) +{ + my ($min, $max) = @_; + + $code.=<<___; + ld $savesp,0($sp) +___ + for (my $i = $min; $i <= $max; $i++) { + my $mult = $max - $i + 1; + $code.=<<___; + lxv $i,-16*$mult($savesp) +___ + } + + $code.=<<___; + mr $sp,$savesp + +___ +} + +sub load_vrs($$) +{ + my ($pointer, $reg_list) = @_; + + for (my $i = 0; $i <= 8; $i++) { + my $offset = $i * 8; + $code.=<<___; + lxsd $reg_list->[$i],$offset($pointer) +___ + } + + $code.=<<___; + +___ +} + +sub store_vrs($$) +{ + my ($pointer, $reg_list) = @_; + + for (my $i = 0; $i <= 8; $i++) { + my $offset = $i * 16; + $code.=<<___; + stxv $reg_list->[$i],$offset($pointer) +___ + } + + $code.=<<___; + +___ +} + +$code.=<<___; +.text + +___ + +{ + # mul/square common + my ($t1, $t2, $t3, $t4) = ("v33", "v34", "v44", "v54"); + my ($zero, $one) = ("r8", "r9"); + my @out = map("v$_",(55..63)); + + { + # + # p521_felem_mul + # + + my ($in1p, $in2p) = ("r4", "r5"); + my @in1 = map("v$_",(45..53)); + my @in2 = map("v$_",(35..43)); + + startproc("p521_felem_mul"); + + push_vrs(52, 63); + + $code.=<<___; + vspltisw $vzero,0 + +___ + + load_vrs($in1p, \@in1); + load_vrs($in2p, \@in2); + + $code.=<<___; + vmsumudm $out[0],$in1[0],$in2[0],$vzero + + xxpermdi $t1,$in1[0],$in1[1],0b00 + xxpermdi $t2,$in2[1],$in2[0],0b00 + vmsumudm $out[1],$t1,$t2,$vzero + + xxpermdi $t2,$in2[2],$in2[1],0b00 + vmsumudm $out[2],$t1,$t2,$vzero + vmsumudm $out[2],$in1[2],$in2[0],$out[2] + + xxpermdi $t2,$in2[3],$in2[2],0b00 + vmsumudm $out[3],$t1,$t2,$vzero + xxpermdi $t3,$in1[2],$in1[3],0b00 + xxpermdi $t4,$in2[1],$in2[0],0b00 + vmsumudm $out[3],$t3,$t4,$out[3] + + xxpermdi $t2,$in2[4],$in2[3],0b00 + vmsumudm $out[4],$t1,$t2,$vzero + xxpermdi $t4,$in2[2],$in2[1],0b00 + vmsumudm $out[4],$t3,$t4,$out[4] + vmsumudm $out[4],$in1[4],$in2[0],$out[4] + + xxpermdi $t2,$in2[5],$in2[4],0b00 + vmsumudm $out[5],$t1,$t2,$vzero + xxpermdi $t4,$in2[3],$in2[2],0b00 + vmsumudm $out[5],$t3,$t4,$out[5] + + xxpermdi $t2,$in2[6],$in2[5],0b00 + vmsumudm $out[6],$t1,$t2,$vzero + xxpermdi $t4,$in2[4],$in2[3],0b00 + vmsumudm $out[6],$t3,$t4,$out[6] + + xxpermdi $t2,$in2[7],$in2[6],0b00 + vmsumudm $out[7],$t1,$t2,$vzero + xxpermdi $t4,$in2[5],$in2[4],0b00 + vmsumudm $out[7],$t3,$t4,$out[7] + + xxpermdi $t2,$in2[8],$in2[7],0b00 + vmsumudm $out[8],$t1,$t2,$vzero + xxpermdi $t4,$in2[6],$in2[5],0b00 + vmsumudm $out[8],$t3,$t4,$out[8] + + xxpermdi $t1,$in1[4],$in1[5],0b00 + xxpermdi $t2,$in2[1],$in2[0],0b00 + vmsumudm $out[5],$t1,$t2,$out[5] + + xxpermdi $t2,$in2[2],$in2[1],0b00 + vmsumudm $out[6],$t1,$t2,$out[6] + vmsumudm $out[6],$in1[6],$in2[0],$out[6] + + xxpermdi $t2,$in2[3],$in2[2],0b00 + vmsumudm $out[7],$t1,$t2,$out[7] + xxpermdi $t3,$in1[6],$in1[7],0b00 + xxpermdi $t4,$in2[1],$in2[0],0b00 + vmsumudm $out[7],$t3,$t4,$out[7] + + xxpermdi $t2,$in2[4],$in2[3],0b00 + vmsumudm $out[8],$t1,$t2,$out[8] + xxpermdi $t4,$in2[2],$in2[1],0b00 + vmsumudm $out[8],$t3,$t4,$out[8] + vmsumudm $out[8],$in1[8],$in2[0],$out[8] + + li $zero,0 + li $one,1 + mtvsrdd $t1,$one,$zero +___ + + for (my $i = 0; $i <= 8; $i++) { + $code.=<<___; + vsld $in2[$i],$in2[$i],$t1 +___ + } + + $code.=<<___; + + vmsumudm $out[7],$in1[8],$in2[8],$out[7] + + xxpermdi $t2,$in2[8],$in2[7],0b00 + xxpermdi $t1,$in1[7],$in1[8],0b00 + vmsumudm $out[6],$t1,$t2,$out[6] + + xxpermdi $t1,$in1[6],$in1[7],0b00 + vmsumudm $out[5],$t1,$t2,$out[5] + vmsumudm $out[5],$in1[8],$in2[6],$out[5] + + xxpermdi $t1,$in1[5],$in1[6],0b00 + vmsumudm $out[4],$t1,$t2,$out[4] + xxpermdi $t4,$in2[6],$in2[5],0b00 + xxpermdi $t3,$in1[7],$in1[8],0b00 + vmsumudm $out[4],$t3,$t4,$out[4] + + xxpermdi $t1,$in1[4],$in1[5],0b00 + vmsumudm $out[3],$t1,$t2,$out[3] + xxpermdi $t3,$in1[6],$in1[7],0b00 + vmsumudm $out[3],$t3,$t4,$out[3] + vmsumudm $out[3],$in1[8],$in2[4],$out[3] + + xxpermdi $t1,$in1[3],$in1[4],0b00 + vmsumudm $out[2],$t1,$t2,$out[2] + xxpermdi $t3,$in1[5],$in1[6],0b00 + vmsumudm $out[2],$t3,$t4,$out[2] + + xxpermdi $t1,$in1[2],$in1[3],0b00 + vmsumudm $out[1],$t1,$t2,$out[1] + xxpermdi $t3,$in1[4],$in1[5],0b00 + vmsumudm $out[1],$t3,$t4,$out[1] + + xxpermdi $t1,$in1[1],$in1[2],0b00 + vmsumudm $out[0],$t1,$t2,$out[0] + xxpermdi $t3,$in1[3],$in1[4],0b00 + vmsumudm $out[0],$t3,$t4,$out[0] + + xxpermdi $t2,$in2[4],$in2[3],0b00 + xxpermdi $t1,$in1[7],$in1[8],0b00 + vmsumudm $out[2],$t1,$t2,$out[2] + + xxpermdi $t1,$in1[6],$in1[7],0b00 + vmsumudm $out[1],$t1,$t2,$out[1] + vmsumudm $out[1],$in1[8],$in2[2],$out[1] + + xxpermdi $t1,$in1[5],$in1[6],0b00 + vmsumudm $out[0],$t1,$t2,$out[0] + xxpermdi $t4,$in2[2],$in2[1],0b00 + xxpermdi $t3,$in1[7],$in1[8],0b00 + vmsumudm $out[0],$t3,$t4,$out[0] + +___ + + store_vrs($outp, \@out); + + pop_vrs(52, 63); + + endproc("p521_felem_mul"); + } + + { + # + # p51_felem_square + # + + my ($inp) = ("r4"); + my @in = map("v$_",(45..53)); + my @inx2 = map("v$_",(35..43)); + + startproc("p521_felem_square"); + + push_vrs(52, 63); + + $code.=<<___; + vspltisw $vzero,0 + +___ + + load_vrs($inp, \@in); + + $code.=<<___; + li $zero,0 + li $one,1 + mtvsrdd $t1,$one,$zero +___ + + for (my $i = 0; $i <= 8; $i++) { + $code.=<<___; + vsld $inx2[$i],$in[$i],$t1 +___ + } + + $code.=<<___; + vmsumudm $out[0],$in[0],$in[0],$vzero + + vmsumudm $out[1],$in[0],$inx2[1],$vzero + + xxpermdi $t1,$in[0],$in[1],0b00 + xxpermdi $t2,$inx2[2],$in[1],0b00 + vmsumudm $out[2],$t1,$t2,$vzero + + xxpermdi $t2,$inx2[3],$inx2[2],0b00 + vmsumudm $out[3],$t1,$t2,$vzero + + xxpermdi $t2,$inx2[4],$inx2[3],0b00 + vmsumudm $out[4],$t1,$t2,$vzero + vmsumudm $out[4],$in[2],$in[2],$out[4] + + xxpermdi $t2,$inx2[5],$inx2[4],0b00 + vmsumudm $out[5],$t1,$t2,$vzero + vmsumudm $out[5],$in[2],$inx2[3],$out[5] + + xxpermdi $t2,$inx2[6],$inx2[5],0b00 + vmsumudm $out[6],$t1,$t2,$vzero + xxpermdi $t3,$in[2],$in[3],0b00 + xxpermdi $t4,$inx2[4],$in[3],0b00 + vmsumudm $out[6],$t3,$t4,$out[6] + + xxpermdi $t2,$inx2[7],$inx2[6],0b00 + vmsumudm $out[7],$t1,$t2,$vzero + xxpermdi $t4,$inx2[5],$inx2[4],0b00 + vmsumudm $out[7],$t3,$t4,$out[7] + + xxpermdi $t2,$inx2[8],$inx2[7],0b00 + vmsumudm $out[8],$t1,$t2,$vzero + xxpermdi $t4,$inx2[6],$inx2[5],0b00 + vmsumudm $out[8],$t3,$t4,$out[8] + vmsumudm $out[8],$in[4],$in[4],$out[8] + + vmsumudm $out[1],$in[5],$inx2[5],$out[1] + + vmsumudm $out[3],$in[6],$inx2[6],$out[3] + + vmsumudm $out[5],$in[7],$inx2[7],$out[5] + + vmsumudm $out[7],$in[8],$inx2[8],$out[7] + + mtvsrdd $t1,$one,$zero +___ + + for (my $i = 5; $i <= 8; $i++) { + $code.=<<___; + vsld $inx2[$i],$inx2[$i],$t1 +___ + } + + $code.=<<___; + + vmsumudm $out[6],$in[7],$inx2[8],$out[6] + + vmsumudm $out[5],$in[6],$inx2[8],$out[5] + + xxpermdi $t2,$inx2[8],$inx2[7],0b00 + xxpermdi $t1,$in[5],$in[6],0b00 + vmsumudm $out[4],$t1,$t2,$out[4] + + xxpermdi $t1,$in[4],$in[5],0b00 + vmsumudm $out[3],$t1,$t2,$out[3] + + xxpermdi $t1,$in[3],$in[4],0b00 + vmsumudm $out[2],$t1,$t2,$out[2] + vmsumudm $out[2],$in[5],$inx2[6],$out[2] + + xxpermdi $t1,$in[2],$in[3],0b00 + vmsumudm $out[1],$t1,$t2,$out[1] + vmsumudm $out[1],$in[4],$inx2[6],$out[1] + + xxpermdi $t1,$in[1],$in[2],0b00 + vmsumudm $out[0],$t1,$t2,$out[0] + xxpermdi $t2,$inx2[6],$inx2[5],0b00 + xxpermdi $t1,$in[3],$in[4],0b00 + vmsumudm $out[0],$t1,$t2,$out[0] + +___ + + store_vrs($outp, \@out); + + pop_vrs(52, 63); + + endproc("p521_felem_square"); + } +} + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +print $code; +close STDOUT or die "error closing STDOUT: $!"; Index: openssl-1.1.1l/crypto/ec/ec_local.h =================================================================== --- openssl-1.1.1l.orig/crypto/ec/ec_local.h +++ openssl-1.1.1l/crypto/ec/ec_local.h @@ -499,6 +499,10 @@ int ec_GF2m_simple_field_div(const EC_GR const BIGNUM *b, BN_CTX *); #ifndef OPENSSL_NO_EC_NISTP_64_GCC_128 +# ifdef B_ENDIAN +# error "Can not enable ec_nistp_64_gcc_128 on big-endian systems" +# endif + /* method functions in ecp_nistp224.c */ int ec_GFp_nistp224_group_init(EC_GROUP *group); int ec_GFp_nistp224_group_set_curve(EC_GROUP *group, const BIGNUM *p, Index: openssl-1.1.1l/crypto/ec/curve448/arch_32/f_impl.c =================================================================== --- openssl-1.1.1l.orig/crypto/ec/curve448/arch_32/f_impl.c +++ openssl-1.1.1l/crypto/ec/curve448/arch_32/f_impl.c @@ -10,7 +10,7 @@ * Originally written by Mike Hamburg */ -#include "field.h" +#include "../field.h" void gf_mul(gf_s * RESTRICT cs, const gf as, const gf bs) { Index: openssl-1.1.1l/crypto/ec/curve448/arch_64/f_impl.c =================================================================== --- /dev/null +++ openssl-1.1.1l/crypto/ec/curve448/arch_64/f_impl.c @@ -0,0 +1,200 @@ +/* + * Copyright 2017-2018 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2014 Cryptography Research, Inc. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + * + * Originally written by Mike Hamburg + */ + +#include "../field.h" + +void gf_mul(gf_s * RESTRICT cs, const gf as, const gf bs) +{ + const uint64_t *a = as->limb, *b = bs->limb; + uint64_t *c = cs->limb; + uint128_t accum0 = 0, accum1 = 0, accum2; + uint64_t mask = (1ULL << 56) - 1; + uint64_t aa[4], bb[4], bbb[4]; + unsigned int i, j; + + for (i = 0; i < 4; i++) { + aa[i] = a[i] + a[i + 4]; + bb[i] = b[i] + b[i + 4]; + bbb[i] = bb[i] + b[i + 4]; + } + + for (i = 0; i < 4; i++) { + accum2 = 0; + + for (j = 0; j <= i; j++) { + accum2 += widemul(a[j], b[i - j]); + accum1 += widemul(aa[j], bb[i - j]); + accum0 += widemul(a[j + 4], b[i - j + 4]); + } + for (; j < 4; j++) { + accum2 += widemul(a[j], b[i - j + 8]); + accum1 += widemul(aa[j], bbb[i - j + 4]); + accum0 += widemul(a[j + 4], bb[i - j + 4]); + } + + accum1 -= accum2; + accum0 += accum2; + + c[i] = ((uint64_t)(accum0)) & mask; + c[i + 4] = ((uint64_t)(accum1)) & mask; + + accum0 >>= 56; + accum1 >>= 56; + } + + accum0 += accum1; + accum0 += c[4]; + accum1 += c[0]; + c[4] = ((uint64_t)(accum0)) & mask; + c[0] = ((uint64_t)(accum1)) & mask; + + accum0 >>= 56; + accum1 >>= 56; + + c[5] += ((uint64_t)(accum0)); + c[1] += ((uint64_t)(accum1)); +} + +void gf_mulw_unsigned(gf_s * RESTRICT cs, const gf as, uint32_t b) +{ + const uint64_t *a = as->limb; + uint64_t *c = cs->limb; + uint128_t accum0 = 0, accum4 = 0; + uint64_t mask = (1ULL << 56) - 1; + int i; + + for (i = 0; i < 4; i++) { + accum0 += widemul(b, a[i]); + accum4 += widemul(b, a[i + 4]); + c[i] = accum0 & mask; + accum0 >>= 56; + c[i + 4] = accum4 & mask; + accum4 >>= 56; + } + + accum0 += accum4 + c[4]; + c[4] = accum0 & mask; + c[5] += accum0 >> 56; + + accum4 += c[0]; + c[0] = accum4 & mask; + c[1] += accum4 >> 56; +} + +void gf_sqr(gf_s * RESTRICT cs, const gf as) +{ + const uint64_t *a = as->limb; + uint64_t *c = cs->limb; + uint128_t accum0 = 0, accum1 = 0, accum2; + uint64_t mask = (1ULL << 56) - 1; + uint64_t aa[4]; + unsigned int i; + + /* For some reason clang doesn't vectorize this without prompting? */ + for (i = 0; i < 4; i++) + aa[i] = a[i] + a[i + 4]; + + accum2 = widemul(a[0], a[3]); + accum0 = widemul(aa[0], aa[3]); + accum1 = widemul(a[4], a[7]); + + accum2 += widemul(a[1], a[2]); + accum0 += widemul(aa[1], aa[2]); + accum1 += widemul(a[5], a[6]); + + accum0 -= accum2; + accum1 += accum2; + + c[3] = ((uint64_t)(accum1)) << 1 & mask; + c[7] = ((uint64_t)(accum0)) << 1 & mask; + + accum0 >>= 55; + accum1 >>= 55; + + accum0 += widemul(2 * aa[1], aa[3]); + accum1 += widemul(2 * a[5], a[7]); + accum0 += widemul(aa[2], aa[2]); + accum1 += accum0; + + accum0 -= widemul(2 * a[1], a[3]); + accum1 += widemul(a[6], a[6]); + + accum2 = widemul(a[0], a[0]); + accum1 -= accum2; + accum0 += accum2; + + accum0 -= widemul(a[2], a[2]); + accum1 += widemul(aa[0], aa[0]); + accum0 += widemul(a[4], a[4]); + + c[0] = ((uint64_t)(accum0)) & mask; + c[4] = ((uint64_t)(accum1)) & mask; + + accum0 >>= 56; + accum1 >>= 56; + + accum2 = widemul(2 * aa[2], aa[3]); + accum0 -= widemul(2 * a[2], a[3]); + accum1 += widemul(2 * a[6], a[7]); + + accum1 += accum2; + accum0 += accum2; + + accum2 = widemul(2 * a[0], a[1]); + accum1 += widemul(2 * aa[0], aa[1]); + accum0 += widemul(2 * a[4], a[5]); + + accum1 -= accum2; + accum0 += accum2; + + c[1] = ((uint64_t)(accum0)) & mask; + c[5] = ((uint64_t)(accum1)) & mask; + + accum0 >>= 56; + accum1 >>= 56; + + accum2 = widemul(aa[3], aa[3]); + accum0 -= widemul(a[3], a[3]); + accum1 += widemul(a[7], a[7]); + + accum1 += accum2; + accum0 += accum2; + + accum2 = widemul(2 * a[0], a[2]); + accum1 += widemul(2 * aa[0], aa[2]); + accum0 += widemul(2 * a[4], a[6]); + + accum2 += widemul(a[1], a[1]); + accum1 += widemul(aa[1], aa[1]); + accum0 += widemul(a[5], a[5]); + + accum1 -= accum2; + accum0 += accum2; + + c[2] = ((uint64_t)(accum0)) & mask; + c[6] = ((uint64_t)(accum1)) & mask; + + accum0 >>= 56; + accum1 >>= 56; + + accum0 += c[3]; + accum1 += c[7]; + c[3] = ((uint64_t)(accum0)) & mask; + c[7] = ((uint64_t)(accum1)) & mask; + + /* we could almost stop here, but it wouldn't be stable, so... */ + + accum0 >>= 56; + accum1 >>= 56; + c[4] += ((uint64_t)(accum0)) + ((uint64_t)(accum1)); + c[0] += ((uint64_t)(accum1)); +} Index: openssl-1.1.1l/Configure =================================================================== --- openssl-1.1.1l.orig/Configure +++ openssl-1.1.1l/Configure @@ -1476,6 +1476,20 @@ if (!$disabled{asm} && !$predefined_C{__ } } +# Check if __SIZEOF_INT128__ is defined by compiler +$config{use_int128} = 0; +{ + my $cc = $config{CROSS_COMPILE}.$config{CC}; + open(PIPE, "$cc -E -dM - </dev/null 2>&1 |"); + while(<PIPE>) { + if (m/__SIZEOF_INT128__/) { + $config{use_int128} = 1; + last; + } + } + close(PIPE); +} + # Deal with bn_ops ################################################### $config{bn_ll} =0;
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
