diff --git a/CMakeLists.txt b/CMakeLists.txt index de126a3f63c..5cb5a6ab951 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -145,6 +145,9 @@ if(BORINGSSL_PREFIX AND BORINGSSL_PREFIX_SYMBOLS AND GO_EXECUTABLE) COMMAND sed -i.bak '/ bignum_/d' ${CMAKE_CURRENT_BINARY_DIR}/symbol_prefix_include/boringssl_prefix_symbols.h COMMAND sed -i.bak '/ bignum_/d' ${CMAKE_CURRENT_BINARY_DIR}/symbol_prefix_include/boringssl_prefix_symbols_asm.h COMMAND sed -i.bak '/ bignum_/d' ${CMAKE_CURRENT_BINARY_DIR}/symbol_prefix_include/boringssl_prefix_symbols_nasm.inc + COMMAND sed -i.bak '/ curve25519_x25519/d' ${CMAKE_CURRENT_BINARY_DIR}/symbol_prefix_include/boringssl_prefix_symbols.h + COMMAND sed -i.bak '/ curve25519_x25519/d' ${CMAKE_CURRENT_BINARY_DIR}/symbol_prefix_include/boringssl_prefix_symbols_asm.h + COMMAND sed -i.bak '/ curve25519_x25519/d' ${CMAKE_CURRENT_BINARY_DIR}/symbol_prefix_include/boringssl_prefix_symbols_nasm.inc DEPENDS util/make_prefix_headers.go ${BORINGSSL_PREFIX_SYMBOLS_PATH}) # add_dependencies needs a target, not a file, so we add an intermediate diff --git a/crypto/curve25519/curve25519.c b/crypto/curve25519/curve25519.c index b0ec2c87d38..9f98c11f6b1 100644 --- a/crypto/curve25519/curve25519.c +++ b/crypto/curve25519/curve25519.c @@ -31,33 +31,226 @@ #include "../internal.h" #include "../fipsmodule/cpucap/internal.h" -// x25519_NEON is defined in asm/x25519-arm.S. #if defined(OPENSSL_ARM) && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_APPLE) -#define BORINGSSL_X25519_NEON +#define BORINGSSL_X25519_NEON_CAPABLE #endif +#if (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) && \ + (defined(OPENSSL_LINUX) || defined(OPENSSL_APPLE)) && \ + !defined(OPENSSL_NO_ASM) && !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_AVX) +#include "../../third_party/s2n-bignum/include/s2n-bignum_aws-lc.h" +#define CURVE25519_S2N_BIGNUM_CAPABLE +#endif -void x25519_NEON(uint8_t out[32], const uint8_t scalar[32], - const uint8_t point[32]); -#if defined(BORINGSSL_X25519_NEON) +OPENSSL_INLINE int x25519_s2n_bignum_capable(void) { +#if defined(CURVE25519_S2N_BIGNUM_CAPABLE) + return 1; +#else + return 0; +#endif +} -OPENSSL_INLINE int curve25519_asm_capable(void) { +OPENSSL_INLINE int x25519_Armv7_neon_capable(void) { +#if defined(BORINGSSL_X25519_NEON_CAPABLE) return CRYPTO_is_NEON_capable(); +#else + return 0; +#endif } -#else +// Stub functions if implementations are not compiled. +// These functions have to abort, otherwise we risk applications assuming they +// did work without actually doing anything. -OPENSSL_INLINE int curve25519_asm_capable(void) { - return 0; +#if !defined(CURVE25519_S2N_BIGNUM_CAPABLE) + +void curve25519_x25519_byte(uint8_t res[32], const uint8_t scalar[32], + const uint8_t point[32]); +void curve25519_x25519_byte_alt(uint8_t res[32], const uint8_t scalar[32], + const uint8_t point[32]); +void curve25519_x25519base_byte(uint8_t res[32], const uint8_t scalar[32]); +void curve25519_x25519base_byte_alt(uint8_t res[32], const uint8_t scalar[32]); + +void curve25519_x25519_byte(uint8_t res[32], const uint8_t scalar[32], + const uint8_t point[32]) { + abort(); +} +void curve25519_x25519_byte_alt(uint8_t res[32], const uint8_t scalar[32], + const uint8_t point[32]) { + abort(); +} +void curve25519_x25519base_byte(uint8_t res[32], const uint8_t scalar[32]) { + abort(); +} +void curve25519_x25519base_byte_alt(uint8_t res[32], const uint8_t scalar[32]) { + abort(); } +#endif // !defined(CURVE25519_S2N_BIGNUM_CAPABLE) + +void x25519_NEON(uint8_t out[32], const uint8_t scalar[32], + const uint8_t point[32]); + +#if !defined(BORINGSSL_X25519_NEON_CAPABLE) void x25519_NEON(uint8_t out[32], const uint8_t scalar[32], const uint8_t point[32]) { abort(); } +#endif // !defined(BORINGSSL_X25519_NEON_CAPABLE) + + +// Run-time detection for each implementation + +OPENSSL_INLINE int x25519_s2n_bignum_alt_capable(void); +OPENSSL_INLINE int x25519_s2n_bignum_no_alt_capable(void); + +// For aarch64, |x25519_s2n_bignum_alt_capable| returns 1 if we categorize the +// CPU as a CPU having a wide multiplier (i.e. "higher" throughput). CPUs with +// this feature are e.g.: AWS Graviton 3 and Apple M1. Return 0 otherwise, so we +// don't match CPUs without wide multipliers. +// +// For x86_64, |x25519_s2n_bignum_alt_capable| always returns 1. If x25519 +// s2n-bignum capable, the x86_64 s2n-bignum-alt version should be supported on +// pretty much any x86_64 CPU. +// +// For all other architectures, return 0. +OPENSSL_INLINE int x25519_s2n_bignum_alt_capable(void) { +#if defined(OPENSSL_X86_64) + return 1; +#elif defined(OPENSSL_AARCH64) + if (CRYPTO_is_ARMv8_wide_multiplier_capable() == 1) { + return 1; + } else { + return 0; + } +#else + return 0; +#endif +} + +// For aarch64, |x25519_s2n_bignum_no_alt_capable| always returns 1. If x25519 +// s2n-bignum capable, the Armv8 s2n-bignum-alt version should be supported on +// pretty much any Armv8 CPU. +// +// For x86_64, |x25519_s2n_bignum_alt_capable| returns 1 if we detect support +// for bmi+adx instruction sets. Return 0 otherwise. +// +// For all other architectures, return 0. +OPENSSL_INLINE int x25519_s2n_bignum_no_alt_capable(void) { +#if defined(OPENSSL_X86_64) + if (CRYPTO_is_BMI2_capable() == 1 && CRYPTO_is_ADX_capable() == 1) { + return 1; + } else { + return 0; + } +#elif defined(OPENSSL_AARCH64) + return 1; +#else + return 0; +#endif +} + + +// Below is the decision logic for which assembly backend implementation +// of x25519 s2n-bignum we should use if x25519 s2n-bignum capable. Currently, +// we support the following implementations. +// +// x86_64: +// - s2n-bignum-no-alt: hardware implementation using bmi2+adx instruction sets +// - s2n-bignum-alt: hardware implementation using standard instructions +// +// aarch64: +// - s2n-bignum-no-alt: hardware implementation for "low" multiplier throughput +// - s2n-bignum-alt: hardware implementation for "high" multiplier throughput +// +// Through experiments we have found that: +// +// For x86_64: bmi+adc will almost always give a performance boost. So, here we +// prefer s2n-bignum-no-alt over s2n-bignum-alt if the former is supported. +// For aarch64: if a wide multiplier is supported, we prefer s2n-bignum-alt over +// s2n-bignum-no-alt if the former is supported. +// x25519_s2n_bignum_alt_capable() specifically looks to match CPUs that have +// wide multipliers. this ensures that s2n-bignum-alt will only be used on +// such CPUs. + +static void x25519_s2n_bignum(uint8_t out_shared_key[32], + const uint8_t private_key[32], const uint8_t peer_public_value[32]) { + + uint8_t private_key_internal_demask[32]; + OPENSSL_memcpy(private_key_internal_demask, private_key, 32); + private_key_internal_demask[0] &= 248; + private_key_internal_demask[31] &= 127; + private_key_internal_demask[31] |= 64; + +#if defined(OPENSSL_X86_64) + + if (x25519_s2n_bignum_no_alt_capable() == 1) { + curve25519_x25519_byte(out_shared_key, private_key_internal_demask, + peer_public_value); + } else if (x25519_s2n_bignum_alt_capable() == 1) { + curve25519_x25519_byte_alt(out_shared_key, private_key_internal_demask, + peer_public_value); + } else { + abort(); + } + +#elif defined(OPENSSL_AARCH64) + + if (x25519_s2n_bignum_alt_capable() == 1) { + curve25519_x25519_byte_alt(out_shared_key, private_key_internal_demask, + peer_public_value); + } else if (x25519_s2n_bignum_no_alt_capable() == 1) { + curve25519_x25519_byte(out_shared_key, private_key_internal_demask, + peer_public_value); + } else { + abort(); + } + +#else + + // Should not call this function unless s2n-bignum is supported. + abort(); + +#endif +} + +static void x25519_s2n_bignum_public_from_private( + uint8_t out_public_value[32], const uint8_t private_key[32]) { + + uint8_t private_key_internal_demask[32]; + OPENSSL_memcpy(private_key_internal_demask, private_key, 32); + private_key_internal_demask[0] &= 248; + private_key_internal_demask[31] &= 127; + private_key_internal_demask[31] |= 64; + +#if defined(OPENSSL_X86_64) + + if (x25519_s2n_bignum_no_alt_capable() == 1) { + curve25519_x25519base_byte(out_public_value, private_key_internal_demask); + } else if (x25519_s2n_bignum_alt_capable() == 1) { + curve25519_x25519base_byte_alt(out_public_value, private_key_internal_demask); + } else { + abort(); + } + +#elif defined(OPENSSL_AARCH64) + + if (x25519_s2n_bignum_alt_capable() == 1) { + curve25519_x25519base_byte_alt(out_public_value, private_key_internal_demask); + } else if (x25519_s2n_bignum_no_alt_capable() == 1) { + curve25519_x25519base_byte(out_public_value, private_key_internal_demask); + } else { + abort(); + } + +#else + + // Should not call this function unless s2n-bignum is supported. + abort(); #endif +} void ED25519_keypair_from_seed(uint8_t out_public_key[32], @@ -189,17 +382,13 @@ int ED25519_verify(const uint8_t *message, size_t message_len, void X25519_public_from_private(uint8_t out_public_value[32], const uint8_t private_key[32]) { - uint8_t e[32]; - OPENSSL_memcpy(e, private_key, 32); - e[0] &= 248; - e[31] &= 127; - e[31] |= 64; - - if (curve25519_asm_capable()) { + if (x25519_s2n_bignum_capable() == 1) { + x25519_s2n_bignum_public_from_private(out_public_value, private_key); + } else if (x25519_Armv7_neon_capable() == 1) { static const uint8_t kMongomeryBasePoint[32] = {9}; x25519_NEON(out_public_value, private_key, kMongomeryBasePoint); } else { - X25519_public_from_private_nohw(out_public_value, e); + x25519_public_from_private_nohw(out_public_value, private_key); } } @@ -231,7 +420,9 @@ int X25519(uint8_t out_shared_key[32], const uint8_t private_key[32], static const uint8_t kZeros[32] = {0}; - if (curve25519_asm_capable()) { + if (x25519_s2n_bignum_capable() == 1) { + x25519_s2n_bignum(out_shared_key, private_key, peer_public_value); + } else if (x25519_Armv7_neon_capable() == 1) { x25519_NEON(out_shared_key, private_key, peer_public_value); } else { x25519_scalar_mult_generic_nohw(out_shared_key, private_key, peer_public_value); diff --git a/crypto/curve25519/curve25519_nohw.c b/crypto/curve25519/curve25519_nohw.c index 3e7b21ec74a..c2adbaba909 100644 --- a/crypto/curve25519/curve25519_nohw.c +++ b/crypto/curve25519/curve25519_nohw.c @@ -1945,11 +1945,17 @@ void x25519_scalar_mult_generic_nohw(uint8_t out[32], fe_tobytes(out, &x2); } -void X25519_public_from_private_nohw(uint8_t out_public_value[32], +void x25519_public_from_private_nohw(uint8_t out_public_value[32], const uint8_t private_key[32]) { + uint8_t e[32]; + OPENSSL_memcpy(e, private_key, 32); + e[0] &= 248; + e[31] &= 127; + e[31] |= 64; + ge_p3 A; - x25519_ge_scalarmult_base(&A, private_key); + x25519_ge_scalarmult_base(&A, e); // We only need the u-coordinate of the curve25519 point. The map is // u=(y+1)/(1-y). Since y=Y/Z, this gives u=(Z+Y)/(Z-Y). diff --git a/crypto/curve25519/internal.h b/crypto/curve25519/internal.h index 247b10e0565..3458c3eb3d2 100644 --- a/crypto/curve25519/internal.h +++ b/crypto/curve25519/internal.h @@ -112,7 +112,7 @@ void x25519_sc_reduce(uint8_t s[64]); void x25519_scalar_mult_generic_nohw(uint8_t out[32], const uint8_t scalar[32], const uint8_t point[32]); -void X25519_public_from_private_nohw(uint8_t out_public_value[32], +void x25519_public_from_private_nohw(uint8_t out_public_value[32], const uint8_t private_key[32]); // Port to internal linkage in curve25519_nohw.c when adding implementation diff --git a/crypto/fipsmodule/CMakeLists.txt b/crypto/fipsmodule/CMakeLists.txt index 5d0f0e740e3..62b49875e27 100644 --- a/crypto/fipsmodule/CMakeLists.txt +++ b/crypto/fipsmodule/CMakeLists.txt @@ -192,6 +192,7 @@ if((ARCH STREQUAL "x86_64" OR ARCH STREQUAL "aarch64") AND p384/bignum_montsqr_p384_alt.S p384/bignum_nonzero_6.S p384/bignum_littleendian_6.S + p521/bignum_add_p521.S p521/bignum_sub_p521.S p521/bignum_neg_p521.S @@ -202,6 +203,7 @@ if((ARCH STREQUAL "x86_64" OR ARCH STREQUAL "aarch64") AND p521/bignum_tolebytes_p521.S p521/bignum_fromlebytes_p521.S ) + if(ARCH STREQUAL "x86_64") # The files below contain the alternative functions for x86_64. # For AArch64, the alternative tomont/deamont functions are @@ -209,7 +211,21 @@ if((ARCH STREQUAL "x86_64" OR ARCH STREQUAL "aarch64") AND # in the corresponding "non-alt" files (bignum__p384.S) list(APPEND S2N_BIGNUM_ASM_SOURCES p384/bignum_tomont_p384_alt.S - p384/bignum_deamont_p384_alt.S) + p384/bignum_deamont_p384_alt.S + curve25519/curve25519_x25519.S + curve25519/curve25519_x25519_alt.S + curve25519/curve25519_x25519base.S + curve25519/curve25519_x25519base_alt.S + ) + elseif(ARCH STREQUAL "aarch64") + # byte-level interface for aarch64 s2n-bignum curve25519 is in + # files with "byte" tags. + list(APPEND S2N_BIGNUM_ASM_SOURCES + curve25519/curve25519_x25519_byte.S + curve25519/curve25519_x25519_byte_alt.S + curve25519/curve25519_x25519base_byte.S + curve25519/curve25519_x25519base_byte_alt.S + ) endif() endif() diff --git a/crypto/fipsmodule/cpucap/internal.h b/crypto/fipsmodule/cpucap/internal.h index 748ea821bb2..03ac640d091 100644 --- a/crypto/fipsmodule/cpucap/internal.h +++ b/crypto/fipsmodule/cpucap/internal.h @@ -198,8 +198,17 @@ OPENSSL_INLINE int CRYPTO_is_ARMv8_GCM_8x_capable(void) { return ((OPENSSL_armcap_P & ARMV8_SHA3) != 0 && ((OPENSSL_armcap_P & ARMV8_NEOVERSE_V1) != 0 || (OPENSSL_armcap_P & ARMV8_APPLE_M1) != 0)); +#endif } + +OPENSSL_INLINE int CRYPTO_is_ARMv8_wide_multiplier_capable(void) { +#if defined(OPENSSL_STATIC_ARMCAP) + return 0; +#else + return (OPENSSL_armcap_P & ARMV8_NEOVERSE_V1) != 0 || + (OPENSSL_armcap_P & ARMV8_APPLE_M1) != 0; #endif +} #endif // OPENSSL_ARM || OPENSSL_AARCH64 diff --git a/crypto/fipsmodule/ec/ec_key.c b/crypto/fipsmodule/ec/ec_key.c index 095bda300d6..ddf33dcaf45 100644 --- a/crypto/fipsmodule/ec/ec_key.c +++ b/crypto/fipsmodule/ec/ec_key.c @@ -332,33 +332,33 @@ static int EVP_EC_KEY_check_fips(EC_KEY *key) { int ret = 0; uint8_t* sig_der = NULL; EVP_PKEY *evp_pkey = EVP_PKEY_new(); - EVP_MD_CTX *ctx = EVP_MD_CTX_new(); + EVP_MD_CTX ctx; + EVP_MD_CTX_init(&ctx); const EVP_MD *hash = EVP_sha256(); size_t sign_len; if (!evp_pkey || - !ctx || !EVP_PKEY_set1_EC_KEY(evp_pkey, key) || - !EVP_DigestSignInit(ctx, NULL, hash, NULL, evp_pkey) || - !EVP_DigestSign(ctx, NULL, &sign_len, msg, msg_len)) { + !EVP_DigestSignInit(&ctx, NULL, hash, NULL, evp_pkey) || + !EVP_DigestSign(&ctx, NULL, &sign_len, msg, msg_len)) { goto err; } sig_der = OPENSSL_malloc(sign_len); if (!sig_der || - !EVP_DigestSign(ctx, sig_der, &sign_len, msg, msg_len)) { + !EVP_DigestSign(&ctx, sig_der, &sign_len, msg, msg_len)) { goto err; } if (boringssl_fips_break_test("ECDSA_PWCT")) { msg[0] = ~msg[0]; } - if (!EVP_DigestVerifyInit(ctx, NULL, hash, NULL, evp_pkey) || - !EVP_DigestVerify(ctx, sig_der, sign_len, msg, msg_len)) { + if (!EVP_DigestVerifyInit(&ctx, NULL, hash, NULL, evp_pkey) || + !EVP_DigestVerify(&ctx, sig_der, sign_len, msg, msg_len)) { goto err; } ret = 1; err: EVP_PKEY_free(evp_pkey); + EVP_MD_CTX_cleanse(&ctx); OPENSSL_free(sig_der); - EVP_MD_CTX_free(ctx); return ret; } diff --git a/crypto/fipsmodule/rsa/rsa.c b/crypto/fipsmodule/rsa/rsa.c index 5eff5cfafc7..8e047b80d42 100644 --- a/crypto/fipsmodule/rsa/rsa.c +++ b/crypto/fipsmodule/rsa/rsa.c @@ -929,6 +929,42 @@ DEFINE_LOCAL_DATA(BIGNUM, g_small_factors) { out->flags = BN_FLG_STATIC_DATA; } +static int EVP_RSA_KEY_check_fips(RSA *key) { + uint8_t msg[1] = {0}; + size_t msg_len = 1; + int ret = 0; + uint8_t* sig_der = NULL; + EVP_PKEY *evp_pkey = EVP_PKEY_new(); + EVP_MD_CTX ctx; + EVP_MD_CTX_init(&ctx); + const EVP_MD *hash = EVP_sha256(); + size_t sign_len; + if (!evp_pkey || + !EVP_PKEY_set1_RSA(evp_pkey, key) || + !EVP_DigestSignInit(&ctx, NULL, hash, NULL, evp_pkey) || + !EVP_DigestSign(&ctx, NULL, &sign_len, msg, msg_len)) { + goto err; + } + sig_der = OPENSSL_malloc(sign_len); + if (!sig_der || + !EVP_DigestSign(&ctx, sig_der, &sign_len, msg, msg_len)) { + goto err; + } + if (boringssl_fips_break_test("RSA_PWCT")) { + msg[0] = ~msg[0]; + } + if (!EVP_DigestVerifyInit(&ctx, NULL, hash, NULL, evp_pkey) || + !EVP_DigestVerify(&ctx, sig_der, sign_len, msg, msg_len)) { + goto err; + } + ret = 1; +err: + EVP_PKEY_free(evp_pkey); + EVP_MD_CTX_cleanse(&ctx); + OPENSSL_free(sig_der); + return ret; +} + int RSA_check_fips(RSA *key) { if (RSA_is_opaque(key)) { // Opaque keys can't be checked. @@ -984,28 +1020,10 @@ int RSA_check_fips(RSA *key) { // section 9.9, it is not known whether |rsa| will be used for signing or // encryption, so either pair-wise consistency self-test is acceptable. We // perform a signing test. - uint8_t data[32] = {0}; - unsigned sig_len = RSA_size(key); - uint8_t *sig = OPENSSL_malloc(sig_len); - if (sig == NULL) { - return 0; - } - - if (!RSA_sign(NID_sha256, data, sizeof(data), sig, &sig_len, key)) { - OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR); + if (!EVP_RSA_KEY_check_fips(key)) { + OPENSSL_PUT_ERROR(EC, RSA_R_PUBLIC_KEY_VALIDATION_FAILED); ret = 0; - goto cleanup; } - if (boringssl_fips_break_test("RSA_PWCT")) { - data[0] = ~data[0]; - } - if (!RSA_verify(NID_sha256, data, sizeof(data), sig, sig_len, key)) { - OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR); - ret = 0; - } - -cleanup: - OPENSSL_free(sig); return ret; } diff --git a/include/openssl/ec.h b/include/openssl/ec.h index fe1e07418ad..6192a4b5944 100644 --- a/include/openssl/ec.h +++ b/include/openssl/ec.h @@ -372,9 +372,11 @@ OPENSSL_EXPORT int EC_GROUP_set_generator(EC_GROUP *group, const BIGNUM *cofactor); -// EC_POINT_point2bn converts an |EC_POINT| to |BIGNUM|. On success, returns the -// BIGNUM pointer supplied, |ret|. Otherwise, it returns NULL on error. The -// |ctx| argument may be used if not NULL. +// EC_POINT_point2bn converts an |EC_POINT| to a |BIGNUM| by serializing the +// point into the X9.62 form given by |form| then interpretting it as a BIGNUM. +// On success, it returns the BIGNUM pointer supplied or, if |ret| is NULL, +// allocates and returns a fresh |BIGNUM|. On error, it returns NULL. The |ctx| +// argument may be used if not NULL. AWS_LC_DEPRECATED OPENSSL_EXPORT BIGNUM *EC_POINT_point2bn( const EC_GROUP *group, const EC_POINT *point, point_conversion_form_t form, BIGNUM *ret, BN_CTX *ctx); diff --git a/tests/ci/cdk/cdk/codebuild/github_ci_linux_x86_omnibus.yaml b/tests/ci/cdk/cdk/codebuild/github_ci_linux_x86_omnibus.yaml index 7e46c1ac5ed..d17168ecce8 100644 --- a/tests/ci/cdk/cdk/codebuild/github_ci_linux_x86_omnibus.yaml +++ b/tests/ci/cdk/cdk/codebuild/github_ci_linux_x86_omnibus.yaml @@ -532,7 +532,7 @@ batch: env: type: LINUX_CONTAINER privileged-mode: true - compute-type: BUILD_GENERAL1_MEDIUM + compute-type: BUILD_GENERAL1_LARGE image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:ubuntu-20.04_clang-7x-bm-framework_latest - identifier: ubuntu1004_gcc4_1x_x86_64_build diff --git a/tests/ci/common_posix_setup.sh b/tests/ci/common_posix_setup.sh index a379945197d..4366c8f75ad 100644 --- a/tests/ci/common_posix_setup.sh +++ b/tests/ci/common_posix_setup.sh @@ -90,7 +90,18 @@ function generate_symbols_file { function verify_symbols_prefixed { go run "$SRC_ROOT"/util/read_symbols.go -out "$BUILD_ROOT"/symbols_final_crypto.txt "$BUILD_ROOT"/crypto/libcrypto.a go run "$SRC_ROOT"/util/read_symbols.go -out "$BUILD_ROOT"/symbols_final_ssl.txt "$BUILD_ROOT"/ssl/libssl.a - cat "$BUILD_ROOT"/symbols_final_crypto.txt "$BUILD_ROOT"/symbols_final_ssl.txt | grep -v -e '^_\?bignum' > "$SRC_ROOT"/symbols_final.txt + # For grep's basic regular expression language the meta-characters (e.g. "?", + # "|", etc.) are interpreted as literal characters. To keep their + # meta-character semantics, they must be escaped with "\". + # Deciphering the pattern "^_\?\(bignum\|curve25519_x25519\)": + # * "^": anchor at start of line. + # * "_\?": might contain underscore. + # * "\(bignum\|curve25519_x25519\)": match string of either "bignum" or "curve25519_x25519". + # Recall that the option "-v" reverse the pattern matching. So, we are really + # filtering out lines that contain either "bignum" or "curve25519_x25519". + cat "$BUILD_ROOT"/symbols_final_crypto.txt "$BUILD_ROOT"/symbols_final_ssl.txt | grep -v -e '^_\?\(bignum\|curve25519_x25519\)' > "$SRC_ROOT"/symbols_final.txt + # Now filter out every line that has the unique prefix $CUSTOM_PREFIX. If we + # have any lines left, then some symbol(s) weren't prefixed, unexpectedly. if [ $(grep -c -v ${CUSTOM_PREFIX} "$SRC_ROOT"/symbols_final.txt) -ne 0 ]; then echo "Symbol(s) missing prefix!" exit 1 diff --git a/third_party/s2n-bignum/include/s2n-bignum_aws-lc.h b/third_party/s2n-bignum/include/s2n-bignum_aws-lc.h index a7a885ad41b..cd1687b9024 100644 --- a/third_party/s2n-bignum/include/s2n-bignum_aws-lc.h +++ b/third_party/s2n-bignum/include/s2n-bignum_aws-lc.h @@ -116,3 +116,17 @@ extern void bignum_fromlebytes_p521(uint64_t z[static 9], const uint8_t x[static // Convert 9-digit 528-bit bignum to little-endian bytes extern void bignum_tolebytes_p521(uint8_t z[static 66], const uint64_t x[static 9]); + +// curve25519_x25519_byte and curve25519_x25519_byte_alt computes the x25519 +// function specified in https://www.rfc-editor.org/rfc/rfc7748. |scalar| is the +// scalar, |point| is the u-coordinate of the elliptic curve +// point. The result, another u-coordinate, is saved in |res|. +extern void curve25519_x25519_byte(uint8_t res[static 32], const uint8_t scalar[static 32], const uint8_t point[static 32]); +extern void curve25519_x25519_byte_alt(uint8_t res[static 32], const uint8_t scalar[static 32], const uint8_t point[static 32]); + +// curve25519_x25519base_byte and curve25519_x25519base_byte_alt computes the +// x25519 function specified in https://www.rfc-editor.org/rfc/rfc7748 using the +// basepoint specified in section 4.1. |scalar| is the scalar. The result, +// another u-coordinate, is saved in |res|. +extern void curve25519_x25519base_byte(uint8_t res[static 32], const uint8_t scalar[static 32]); +extern void curve25519_x25519base_byte_alt(uint8_t res[static 32], const uint8_t scalar[static 32]); diff --git a/tool/speed.cc b/tool/speed.cc index 1c01d9a3c1c..a1c954cbbe5 100644 --- a/tool/speed.cc +++ b/tool/speed.cc @@ -335,7 +335,7 @@ static bool SpeedRSA(const std::string &selected) { return true; } -static bool SpeedRSAKeyGen(const std::string &selected) { +static bool SpeedRSAKeyGen(bool is_fips, const std::string &selected) { // Don't run this by default because it's so slow. if (selected != "RSAKeyGen") { return true; @@ -357,10 +357,24 @@ static bool SpeedRSAKeyGen(const std::string &selected) { BM_NAMESPACE::UniquePtr rsa(RSA_new()); const uint64_t iteration_start = time_now(); - if (!RSA_generate_key_ex(rsa.get(), size, e.get(), nullptr)) { - fprintf(stderr, "RSA_generate_key_ex failed.\n"); - ERR_print_errors_fp(stderr); - return false; + if(is_fips){ +#if !defined(OPENSSL_BENCHMARK) + // RSA_generate_key_fips is AWS-LC specific. + if (!RSA_generate_key_fips(rsa.get(), size, nullptr)) { + fprintf(stderr, "RSA_generate_key_fips failed.\n"); + ERR_print_errors_fp(stderr); + return false; + } +#else + return true; +#endif + } + else { + if (!RSA_generate_key_ex(rsa.get(), size, e.get(), nullptr)) { + fprintf(stderr, "RSA_generate_key_ex failed.\n"); + ERR_print_errors_fp(stderr); + return false; + } } const uint64_t iteration_end = time_now(); @@ -374,8 +388,12 @@ static bool SpeedRSAKeyGen(const std::string &selected) { } std::sort(durations.begin(), durations.end()); + std::string rsa_type = std::string("RSA "); + if (is_fips) { + rsa_type += "FIPS "; + } const std::string description = - std::string("RSA ") + std::to_string(size) + std::string(" key-gen"); + rsa_type + std::to_string(size) + std::string(" key-gen"); const TimeResults results = {num_calls, us}; results.Print(description); const size_t n = durations.size(); @@ -1864,7 +1882,7 @@ static bool SpeedTrustToken(std::string name, const TRUST_TOKEN_METHOD *method, #endif #endif -#if defined(BORINGSSL_FIPS) +#if defined(AWSLC_FIPS) static bool SpeedSelfTest(const std::string &selected) { if (!selected.empty() && selected.find("self-test") == std::string::npos) { return true; @@ -2162,7 +2180,7 @@ bool Speed(const std::vector &args) { !SpeedScrypt(selected) || #endif !SpeedRSA(selected) || - !SpeedRSAKeyGen(selected) + !SpeedRSAKeyGen(false, selected) #if !defined(OPENSSL_BENCHMARK) || !SpeedKEM(selected) || @@ -2183,7 +2201,7 @@ bool Speed(const std::vector &args) { !SpeedAEAD(EVP_aead_aes_128_ccm_bluetooth(), "AEAD-AES-128-CCM-Bluetooth", kTLSADLen, selected) || !Speed25519(selected) || !SpeedSPAKE2(selected) || - !SpeedRSAKeyGen(selected) || + !SpeedRSAKeyGen(true, selected) || !SpeedHRSS(selected) || !SpeedHash(EVP_blake2b256(), "BLAKE2b-256", selected) || !SpeedECKeyGenerateKey(true, selected) || @@ -2205,7 +2223,7 @@ bool Speed(const std::vector &args) { ) { return false; } -#if defined(BORINGSSL_FIPS) +#if defined(AWSLC_FIPS) if (!SpeedSelfTest(selected) || !SpeedJitter(selected)) { return false; diff --git a/util/fipstools/delocate/delocate.peg b/util/fipstools/delocate/delocate.peg index 17af6dae339..423c64b26e1 100644 --- a/util/fipstools/delocate/delocate.peg +++ b/util/fipstools/delocate/delocate.peg @@ -106,10 +106,13 @@ Operator <- [+\-] OffsetOperator <- '+' / '-' / '*' Offset <- '+'? '-'? (("0b" [01]+) / ("0x" [[0-9A-F]]+) / - ([0-9]+ ( OffsetOperator '(' [0-9]+ OffsetOperator [0-9]+ ')' )? / + ([0-9]+ WS OffsetOperator [0-9]+ / + [0-9]+ ( OffsetOperator '(' [0-9]+ OffsetOperator [0-9]+ ')' )? / [0-9]+ ( OffsetOperator [0-9]+ OffsetOperator [0-9]+ )? / [0-9]+ ( OffsetOperator [0-9]+ )? / - '(' [0-9]+ WS? OffsetOperator WS? [0-9]+ ')')![[A-Z]] + '(' [0-9]+ WS? OffsetOperator WS? [0-9]+ ')' OffsetOperator [0-9]+ OffsetOperator [0-9]+ / + '(' [0-9]+ WS? OffsetOperator WS? [0-9]+ ')' / + '(' [0-9]+ WS? OffsetOperator WS? [0-9]+ WS? OffsetOperator WS? [0-9]+')')![[A-Z]] ) Section <- [[A-Z@]]+ SegmentRegister <- '%' [c-gs] 's:' diff --git a/util/fipstools/delocate/delocate.peg.go b/util/fipstools/delocate/delocate.peg.go index 56621f3a892..1111645d004 100644 --- a/util/fipstools/delocate/delocate.peg.go +++ b/util/fipstools/delocate/delocate.peg.go @@ -3,12 +3,12 @@ package main // Code generated by ./peg/peg delocate.peg DO NOT EDIT. import ( + "bytes" "fmt" "io" "os" "sort" "strconv" - "strings" ) const endSymbol rune = 1114112 @@ -244,12 +244,13 @@ func (t *tokens32) Tokens() []token32 { } type Asm struct { - Buffer string - buffer []rune - rules [56]func() bool - parse func(rule ...int) error - reset func() - Pretty bool + Buffer string + buffer []rune + rules [56]func() bool + parse func(rule ...int) error + reset func() + Pretty bool + disableMemoize bool tokens32 } @@ -334,9 +335,9 @@ func (p *Asm) WriteSyntaxTree(w io.Writer) { } func (p *Asm) SprintSyntaxTree() string { - var bldr strings.Builder - p.WriteSyntaxTree(&bldr) - return bldr.String() + var b bytes.Buffer + p.WriteSyntaxTree(&b) + return b.String() } func Pretty(pretty bool) func(*Asm) error { @@ -352,11 +353,30 @@ func Size(size int) func(*Asm) error { return nil } } + +func DisableMemoize() func(*Asm) error { + return func(p *Asm) error { + p.disableMemoize = true + return nil + } +} + +type memo struct { + Matched bool + Partial []token32 +} + +type memoKey struct { + Rule uint32 + Position uint32 +} + func (p *Asm) Init(options ...func(*Asm) error) error { var ( max token32 position, tokenIndex uint32 buffer []rune + memoization map[memoKey]memo ) for _, option := range options { err := option(p) @@ -367,7 +387,7 @@ func (p *Asm) Init(options ...func(*Asm) error) error { p.reset = func() { max = token32{} position, tokenIndex = 0, 0 - + memoization = make(map[memoKey]memo) p.buffer = []rune(p.Buffer) if len(p.buffer) == 0 || p.buffer[len(p.buffer)-1] != endSymbol { p.buffer = append(p.buffer, endSymbol) @@ -400,6 +420,34 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } } + memoize := func(rule uint32, begin uint32, tokenIndexStart uint32, matched bool) { + if p.disableMemoize { + return + } + key := memoKey{rule, begin} + if !matched { + memoization[key] = memo{Matched: false} + } else { + t := tree.tree[tokenIndexStart:tokenIndex] + tokenCopy := make([]token32, len(t)) + copy(tokenCopy, t) + memoization[key] = memo{Matched: true, Partial: tokenCopy} + } + } + + memoizedResult := func(m memo) bool { + if !m.Matched { + return false + } + tree.tree = append(tree.tree[:tokenIndex], m.Partial...) + tokenIndex += uint32(len(m.Partial)) + position = m.Partial[len(m.Partial)-1].end + if tree.tree[tokenIndex-1].begin != position && position > max.end { + max = tree.tree[tokenIndex-1] + } + return true + } + matchDot := func() bool { if buffer[position] != endSymbol { position++ @@ -428,6 +476,9 @@ func (p *Asm) Init(options ...func(*Asm) error) error { nil, /* 0 AsmFile <- <(Statement* !.)> */ func() bool { + if memoized, ok := memoization[memoKey{0, position}]; ok { + return memoizedResult(memoized) + } position0, tokenIndex0 := position, tokenIndex { position1 := position @@ -452,13 +503,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleAsmFile, position1) } + memoize(0, position0, tokenIndex0, true) return true l0: + memoize(0, position0, tokenIndex0, false) position, tokenIndex = position0, tokenIndex0 return false }, /* 1 Statement <- <(WS? (Label / ((GlobalDirective / LocationDirective / LabelContainingDirective / Instruction / Directive / Comment / ) WS? ((Comment? '\n') / ';'))))> */ func() bool { + if memoized, ok := memoization[memoKey{1, position}]; ok { + return memoizedResult(memoized) + } position5, tokenIndex5 := position, tokenIndex { position6 := position @@ -559,13 +615,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l9: add(ruleStatement, position6) } + memoize(1, position5, tokenIndex5, true) return true l5: + memoize(1, position5, tokenIndex5, false) position, tokenIndex = position5, tokenIndex5 return false }, /* 2 GlobalDirective <- <((('.' ('g' / 'G') ('l' / 'L') ('o' / 'O') ('b' / 'B') ('a' / 'A') ('l' / 'L')) / ('.' ('g' / 'G') ('l' / 'L') ('o' / 'O') ('b' / 'B') ('l' / 'L'))) WS SymbolName)> */ func() bool { + if memoized, ok := memoization[memoKey{2, position}]; ok { + return memoizedResult(memoized) + } position24, tokenIndex24 := position, tokenIndex { position25 := position @@ -757,13 +818,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleGlobalDirective, position25) } + memoize(2, position24, tokenIndex24, true) return true l24: + memoize(2, position24, tokenIndex24, false) position, tokenIndex = position24, tokenIndex24 return false }, /* 3 Directive <- <('.' DirectiveName (WS Args)?)> */ func() bool { + if memoized, ok := memoization[memoKey{3, position}]; ok { + return memoizedResult(memoized) + } position50, tokenIndex50 := position, tokenIndex { position51 := position @@ -789,13 +855,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l53: add(ruleDirective, position51) } + memoize(3, position50, tokenIndex50, true) return true l50: + memoize(3, position50, tokenIndex50, false) position, tokenIndex = position50, tokenIndex50 return false }, /* 4 DirectiveName <- <([a-z] / [A-Z] / ([0-9] / [0-9]) / '_')+> */ func() bool { + if memoized, ok := memoization[memoKey{4, position}]; ok { + return memoizedResult(memoized) + } position54, tokenIndex54 := position, tokenIndex { position55 := position @@ -888,13 +959,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleDirectiveName, position55) } + memoize(4, position54, tokenIndex54, true) return true l54: + memoize(4, position54, tokenIndex54, false) position, tokenIndex = position54, tokenIndex54 return false }, /* 5 LocationDirective <- <(FileDirective / LocDirective)> */ func() bool { + if memoized, ok := memoization[memoKey{5, position}]; ok { + return memoizedResult(memoized) + } position70, tokenIndex70 := position, tokenIndex { position71 := position @@ -913,13 +989,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l72: add(ruleLocationDirective, position71) } + memoize(5, position70, tokenIndex70, true) return true l70: + memoize(5, position70, tokenIndex70, false) position, tokenIndex = position70, tokenIndex70 return false }, /* 6 FileDirective <- <('.' ('f' / 'F') ('i' / 'I') ('l' / 'L') ('e' / 'E') WS (!('#' / '\n') .)+)> */ func() bool { + if memoized, ok := memoization[memoKey{6, position}]; ok { + return memoizedResult(memoized) + } position74, tokenIndex74 := position, tokenIndex { position75 := position @@ -1047,13 +1128,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleFileDirective, position75) } + memoize(6, position74, tokenIndex74, true) return true l74: + memoize(6, position74, tokenIndex74, false) position, tokenIndex = position74, tokenIndex74 return false }, /* 7 LocDirective <- <('.' ('l' / 'L') ('o' / 'O') ('c' / 'C') WS (!('#' / '/' / '\n') .)+)> */ func() bool { + if memoized, ok := memoization[memoKey{7, position}]; ok { + return memoizedResult(memoized) + } position92, tokenIndex92 := position, tokenIndex { position93 := position @@ -1180,13 +1266,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleLocDirective, position93) } + memoize(7, position92, tokenIndex92, true) return true l92: + memoize(7, position92, tokenIndex92, false) position, tokenIndex = position92, tokenIndex92 return false }, /* 8 Args <- <(Arg (WS? ',' WS? Arg)*)> */ func() bool { + if memoized, ok := memoization[memoKey{8, position}]; ok { + return memoizedResult(memoized) + } position110, tokenIndex110 := position, tokenIndex { position111 := position @@ -1229,13 +1320,19 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleArgs, position111) } + memoize(8, position110, tokenIndex110, true) return true l110: + memoize(8, position110, tokenIndex110, false) position, tokenIndex = position110, tokenIndex110 return false }, /* 9 Arg <- <(QuotedArg / ([0-9] / [0-9] / ([a-z] / [A-Z]) / '%' / '+' / '-' / '*' / '_' / '@' / '.')*)> */ func() bool { + if memoized, ok := memoization[memoKey{9, position}]; ok { + return memoizedResult(memoized) + } + position118, tokenIndex118 := position, tokenIndex { position119 := position { @@ -1339,10 +1436,14 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l120: add(ruleArg, position119) } + memoize(9, position118, tokenIndex118, true) return true }, /* 10 QuotedArg <- <('"' QuotedText '"')> */ func() bool { + if memoized, ok := memoization[memoKey{10, position}]; ok { + return memoizedResult(memoized) + } position136, tokenIndex136 := position, tokenIndex { position137 := position @@ -1359,13 +1460,19 @@ func (p *Asm) Init(options ...func(*Asm) error) error { position++ add(ruleQuotedArg, position137) } + memoize(10, position136, tokenIndex136, true) return true l136: + memoize(10, position136, tokenIndex136, false) position, tokenIndex = position136, tokenIndex136 return false }, /* 11 QuotedText <- <(EscapedChar / (!'"' .))*> */ func() bool { + if memoized, ok := memoization[memoKey{11, position}]; ok { + return memoizedResult(memoized) + } + position138, tokenIndex138 := position, tokenIndex { position139 := position l140: @@ -1400,10 +1507,14 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleQuotedText, position139) } + memoize(11, position138, tokenIndex138, true) return true }, /* 12 LabelContainingDirective <- <(LabelContainingDirectiveName WS SymbolArgs)> */ func() bool { + if memoized, ok := memoization[memoKey{12, position}]; ok { + return memoizedResult(memoized) + } position145, tokenIndex145 := position, tokenIndex { position146 := position @@ -1418,13 +1529,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleLabelContainingDirective, position146) } + memoize(12, position145, tokenIndex145, true) return true l145: + memoize(12, position145, tokenIndex145, false) position, tokenIndex = position145, tokenIndex145 return false }, /* 13 LabelContainingDirectiveName <- <(('.' ('x' / 'X') ('w' / 'W') ('o' / 'O') ('r' / 'R') ('d' / 'D')) / ('.' ('w' / 'W') ('o' / 'O') ('r' / 'R') ('d' / 'D')) / ('.' ('l' / 'L') ('o' / 'O') ('n' / 'N') ('g' / 'G')) / ('.' ('s' / 'S') ('e' / 'E') ('t' / 'T')) / ('.' ('b' / 'B') ('y' / 'Y') ('t' / 'T') ('e' / 'E')) / ('.' '8' ('b' / 'B') ('y' / 'Y') ('t' / 'T') ('e' / 'E')) / ('.' '4' ('b' / 'B') ('y' / 'Y') ('t' / 'T') ('e' / 'E')) / ('.' ('q' / 'Q') ('u' / 'U') ('a' / 'A') ('d' / 'D')) / ('.' ('t' / 'T') ('c' / 'C')) / ('.' ('l' / 'L') ('o' / 'O') ('c' / 'C') ('a' / 'A') ('l' / 'L') ('e' / 'E') ('n' / 'N') ('t' / 'T') ('r' / 'R') ('y' / 'Y')) / ('.' ('s' / 'S') ('i' / 'I') ('z' / 'Z') ('e' / 'E')) / ('.' ('t' / 'T') ('y' / 'Y') ('p' / 'P') ('e' / 'E')) / ('.' ('u' / 'U') ('l' / 'L') ('e' / 'E') ('b' / 'B') '1' '2' '8') / ('.' ('s' / 'S') ('l' / 'L') ('e' / 'E') ('b' / 'B') '1' '2' '8'))> */ func() bool { + if memoized, ok := memoization[memoKey{13, position}]; ok { + return memoizedResult(memoized) + } position147, tokenIndex147 := position, tokenIndex { position148 := position @@ -2461,13 +2577,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l149: add(ruleLabelContainingDirectiveName, position148) } + memoize(13, position147, tokenIndex147, true) return true l147: + memoize(13, position147, tokenIndex147, false) position, tokenIndex = position147, tokenIndex147 return false }, /* 14 SymbolArgs <- <(SymbolArg (WS? ',' WS? SymbolArg)*)> */ func() bool { + if memoized, ok := memoization[memoKey{14, position}]; ok { + return memoizedResult(memoized) + } position283, tokenIndex283 := position, tokenIndex { position284 := position @@ -2510,13 +2631,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleSymbolArgs, position284) } + memoize(14, position283, tokenIndex283, true) return true l283: + memoize(14, position283, tokenIndex283, false) position, tokenIndex = position283, tokenIndex283 return false }, /* 15 SymbolShift <- <((('<' '<') / ('>' '>')) WS? [0-9]+)> */ func() bool { + if memoized, ok := memoization[memoKey{15, position}]; ok { + return memoizedResult(memoized) + } position291, tokenIndex291 := position, tokenIndex { position292 := position @@ -2570,13 +2696,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleSymbolShift, position292) } + memoize(15, position291, tokenIndex291, true) return true l291: + memoize(15, position291, tokenIndex291, false) position, tokenIndex = position291, tokenIndex291 return false }, /* 16 SymbolArg <- <((OpenParen WS?)? (Offset / SymbolType / ((Offset / LocalSymbol / SymbolName / Dot) (WS? Operator WS? (Offset / LocalSymbol / SymbolName))*) / (LocalLabelRef WS? Operator WS? LocalLabelRef) / (LocalSymbol TCMarker?) / (SymbolName Offset) / (SymbolName TCMarker?)) (WS? CloseParen)? (WS? SymbolShift)?)> */ func() bool { + if memoized, ok := memoization[memoKey{16, position}]; ok { + return memoizedResult(memoized) + } position299, tokenIndex299 := position, tokenIndex { position300 := position @@ -2805,13 +2936,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l338: add(ruleSymbolArg, position300) } + memoize(16, position299, tokenIndex299, true) return true l299: + memoize(16, position299, tokenIndex299, false) position, tokenIndex = position299, tokenIndex299 return false }, /* 17 OpenParen <- <'('> */ func() bool { + if memoized, ok := memoization[memoKey{17, position}]; ok { + return memoizedResult(memoized) + } position341, tokenIndex341 := position, tokenIndex { position342 := position @@ -2821,13 +2957,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { position++ add(ruleOpenParen, position342) } + memoize(17, position341, tokenIndex341, true) return true l341: + memoize(17, position341, tokenIndex341, false) position, tokenIndex = position341, tokenIndex341 return false }, /* 18 CloseParen <- <')'> */ func() bool { + if memoized, ok := memoization[memoKey{18, position}]; ok { + return memoizedResult(memoized) + } position343, tokenIndex343 := position, tokenIndex { position344 := position @@ -2837,13 +2978,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { position++ add(ruleCloseParen, position344) } + memoize(18, position343, tokenIndex343, true) return true l343: + memoize(18, position343, tokenIndex343, false) position, tokenIndex = position343, tokenIndex343 return false }, /* 19 SymbolType <- <(('@' / '%') (('f' 'u' 'n' 'c' 't' 'i' 'o' 'n') / ('o' 'b' 'j' 'e' 'c' 't')))> */ func() bool { + if memoized, ok := memoization[memoKey{19, position}]; ok { + return memoizedResult(memoized) + } position345, tokenIndex345 := position, tokenIndex { position346 := position @@ -2927,13 +3073,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l349: add(ruleSymbolType, position346) } + memoize(19, position345, tokenIndex345, true) return true l345: + memoize(19, position345, tokenIndex345, false) position, tokenIndex = position345, tokenIndex345 return false }, /* 20 Dot <- <'.'> */ func() bool { + if memoized, ok := memoization[memoKey{20, position}]; ok { + return memoizedResult(memoized) + } position351, tokenIndex351 := position, tokenIndex { position352 := position @@ -2943,13 +3094,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { position++ add(ruleDot, position352) } + memoize(20, position351, tokenIndex351, true) return true l351: + memoize(20, position351, tokenIndex351, false) position, tokenIndex = position351, tokenIndex351 return false }, /* 21 TCMarker <- <('[' 'T' 'C' ']')> */ func() bool { + if memoized, ok := memoization[memoKey{21, position}]; ok { + return memoizedResult(memoized) + } position353, tokenIndex353 := position, tokenIndex { position354 := position @@ -2971,13 +3127,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { position++ add(ruleTCMarker, position354) } + memoize(21, position353, tokenIndex353, true) return true l353: + memoize(21, position353, tokenIndex353, false) position, tokenIndex = position353, tokenIndex353 return false }, /* 22 EscapedChar <- <('\\' .)> */ func() bool { + if memoized, ok := memoization[memoKey{22, position}]; ok { + return memoizedResult(memoized) + } position355, tokenIndex355 := position, tokenIndex { position356 := position @@ -2990,13 +3151,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleEscapedChar, position356) } + memoize(22, position355, tokenIndex355, true) return true l355: + memoize(22, position355, tokenIndex355, false) position, tokenIndex = position355, tokenIndex355 return false }, /* 23 WS <- <(' ' / '\t')+> */ func() bool { + if memoized, ok := memoization[memoKey{23, position}]; ok { + return memoizedResult(memoized) + } position357, tokenIndex357 := position, tokenIndex { position358 := position @@ -3039,13 +3205,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleWS, position358) } + memoize(23, position357, tokenIndex357, true) return true l357: + memoize(23, position357, tokenIndex357, false) position, tokenIndex = position357, tokenIndex357 return false }, /* 24 Comment <- <((('/' '/') / '#') (!'\n' .)*)> */ func() bool { + if memoized, ok := memoization[memoKey{24, position}]; ok { + return memoizedResult(memoized) + } position365, tokenIndex365 := position, tokenIndex { position366 := position @@ -3090,13 +3261,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleComment, position366) } + memoize(24, position365, tokenIndex365, true) return true l365: + memoize(24, position365, tokenIndex365, false) position, tokenIndex = position365, tokenIndex365 return false }, /* 25 Label <- <((LocalSymbol / LocalLabel / SymbolName) ':')> */ func() bool { + if memoized, ok := memoization[memoKey{25, position}]; ok { + return memoizedResult(memoized) + } position372, tokenIndex372 := position, tokenIndex { position373 := position @@ -3125,13 +3301,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { position++ add(ruleLabel, position373) } + memoize(25, position372, tokenIndex372, true) return true l372: + memoize(25, position372, tokenIndex372, false) position, tokenIndex = position372, tokenIndex372 return false }, /* 26 SymbolName <- <(([a-z] / [A-Z] / '.' / '_') ([a-z] / [A-Z] / '.' / ([0-9] / [0-9]) / '$' / '_')*)> */ func() bool { + if memoized, ok := memoization[memoKey{26, position}]; ok { + return memoizedResult(memoized) + } position377, tokenIndex377 := position, tokenIndex { position378 := position @@ -3227,13 +3408,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleSymbolName, position378) } + memoize(26, position377, tokenIndex377, true) return true l377: + memoize(26, position377, tokenIndex377, false) position, tokenIndex = position377, tokenIndex377 return false }, /* 27 LocalSymbol <- <('.' 'L' ([a-z] / [A-Z] / ([a-z] / [A-Z]) / '.' / ([0-9] / [0-9]) / '$' / '_')+)> */ func() bool { + if memoized, ok := memoization[memoKey{27, position}]; ok { + return memoizedResult(memoized) + } position393, tokenIndex393 := position, tokenIndex { position394 := position @@ -3398,13 +3584,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleLocalSymbol, position394) } + memoize(27, position393, tokenIndex393, true) return true l393: + memoize(27, position393, tokenIndex393, false) position, tokenIndex = position393, tokenIndex393 return false }, /* 28 LocalLabel <- <([0-9] ([0-9] / '$')*)> */ func() bool { + if memoized, ok := memoization[memoKey{28, position}]; ok { + return memoizedResult(memoized) + } position419, tokenIndex419 := position, tokenIndex { position420 := position @@ -3436,13 +3627,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleLocalLabel, position420) } + memoize(28, position419, tokenIndex419, true) return true l419: + memoize(28, position419, tokenIndex419, false) position, tokenIndex = position419, tokenIndex419 return false }, /* 29 LocalLabelRef <- <([0-9] ([0-9] / '$')* ('b' / 'f'))> */ func() bool { + if memoized, ok := memoization[memoKey{29, position}]; ok { + return memoizedResult(memoized) + } position425, tokenIndex425 := position, tokenIndex { position426 := position @@ -3489,13 +3685,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l431: add(ruleLocalLabelRef, position426) } + memoize(29, position425, tokenIndex425, true) return true l425: + memoize(29, position425, tokenIndex425, false) position, tokenIndex = position425, tokenIndex425 return false }, /* 30 Instruction <- <(InstructionName (WS InstructionArg (WS? ','? WS? InstructionArg)*)?)> */ func() bool { + if memoized, ok := memoization[memoKey{30, position}]; ok { + return memoizedResult(memoized) + } position433, tokenIndex433 := position, tokenIndex { position434 := position @@ -3558,13 +3759,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l436: add(ruleInstruction, position434) } + memoize(30, position433, tokenIndex433, true) return true l433: + memoize(30, position433, tokenIndex433, false) position, tokenIndex = position433, tokenIndex433 return false }, /* 31 InstructionName <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / '.' / ([0-9] / [0-9]))* ('.' / '+' / '-')?)> */ func() bool { + if memoized, ok := memoization[memoKey{31, position}]; ok { + return memoizedResult(memoized) + } position445, tokenIndex445 := position, tokenIndex { position446 := position @@ -3661,13 +3867,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l458: add(ruleInstructionName, position446) } + memoize(31, position445, tokenIndex445, true) return true l445: + memoize(31, position445, tokenIndex445, false) position, tokenIndex = position445, tokenIndex445 return false }, /* 32 InstructionArg <- <(IndirectionIndicator? (ARMConstantTweak / RegisterOrConstant / LocalLabelRef / TOCRefHigh / TOCRefLow / GOTLocation / GOTSymbolOffset / MemoryRef) AVX512Token*)> */ func() bool { + if memoized, ok := memoization[memoKey{32, position}]; ok { + return memoizedResult(memoized) + } position462, tokenIndex462 := position, tokenIndex { position463 := position @@ -3742,13 +3953,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleInstructionArg, position463) } + memoize(32, position462, tokenIndex462, true) return true l462: + memoize(32, position462, tokenIndex462, false) position, tokenIndex = position462, tokenIndex462 return false }, /* 33 GOTLocation <- <('$' '_' 'G' 'L' 'O' 'B' 'A' 'L' '_' 'O' 'F' 'F' 'S' 'E' 'T' '_' 'T' 'A' 'B' 'L' 'E' '_' '-' LocalSymbol)> */ func() bool { + if memoized, ok := memoization[memoKey{33, position}]; ok { + return memoizedResult(memoized) + } position476, tokenIndex476 := position, tokenIndex { position477 := position @@ -3849,13 +4065,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleGOTLocation, position477) } + memoize(33, position476, tokenIndex476, true) return true l476: + memoize(33, position476, tokenIndex476, false) position, tokenIndex = position476, tokenIndex476 return false }, /* 34 GOTSymbolOffset <- <(('$' SymbolName ('@' 'G' 'O' 'T') ('O' 'F' 'F')?) / (':' ('g' / 'G') ('o' / 'O') ('t' / 'T') ':' SymbolName))> */ func() bool { + if memoized, ok := memoization[memoKey{34, position}]; ok { + return memoizedResult(memoized) + } position478, tokenIndex478 := position, tokenIndex { position479 := position @@ -3966,13 +4187,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l480: add(ruleGOTSymbolOffset, position479) } + memoize(34, position478, tokenIndex478, true) return true l478: + memoize(34, position478, tokenIndex478, false) position, tokenIndex = position478, tokenIndex478 return false }, /* 35 AVX512Token <- <(WS? '{' '%'? ([0-9] / [a-z])* '}')> */ func() bool { + if memoized, ok := memoization[memoKey{35, position}]; ok { + return memoizedResult(memoized) + } position490, tokenIndex490 := position, tokenIndex { position491 := position @@ -4029,13 +4255,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { position++ add(ruleAVX512Token, position491) } + memoize(35, position490, tokenIndex490, true) return true l490: + memoize(35, position490, tokenIndex490, false) position, tokenIndex = position490, tokenIndex490 return false }, /* 36 TOCRefHigh <- <('.' 'T' 'O' 'C' '.' '-' (('0' 'b') / ('.' 'L' ([a-z] / [A-Z] / '_' / [0-9])+)) ('@' ('h' / 'H') ('a' / 'A')))> */ func() bool { + if memoized, ok := memoization[memoKey{36, position}]; ok { + return memoizedResult(memoized) + } position500, tokenIndex500 := position, tokenIndex { position501 := position @@ -4187,13 +4418,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l516: add(ruleTOCRefHigh, position501) } + memoize(36, position500, tokenIndex500, true) return true l500: + memoize(36, position500, tokenIndex500, false) position, tokenIndex = position500, tokenIndex500 return false }, /* 37 TOCRefLow <- <('.' 'T' 'O' 'C' '.' '-' (('0' 'b') / ('.' 'L' ([a-z] / [A-Z] / '_' / [0-9])+)) ('@' ('l' / 'L')))> */ func() bool { + if memoized, ok := memoization[memoKey{37, position}]; ok { + return memoizedResult(memoized) + } position518, tokenIndex518 := position, tokenIndex { position519 := position @@ -4330,13 +4566,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l532: add(ruleTOCRefLow, position519) } + memoize(37, position518, tokenIndex518, true) return true l518: + memoize(37, position518, tokenIndex518, false) position, tokenIndex = position518, tokenIndex518 return false }, /* 38 IndirectionIndicator <- <'*'> */ func() bool { + if memoized, ok := memoization[memoKey{38, position}]; ok { + return memoizedResult(memoized) + } position534, tokenIndex534 := position, tokenIndex { position535 := position @@ -4346,13 +4587,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { position++ add(ruleIndirectionIndicator, position535) } + memoize(38, position534, tokenIndex534, true) return true l534: + memoize(38, position534, tokenIndex534, false) position, tokenIndex = position534, tokenIndex534 return false }, /* 39 RegisterOrConstant <- <((('%' ([a-z] / [A-Z]) ([a-z] / [A-Z] / ([0-9] / [0-9]))*) / ('$' [0-9]+ WS? '*' WS? '(' [0-9]+ WS? '-' WS? [0-9]+ ')') / ('$'? ((Offset Offset) / Offset)) / ('#' Offset ('*' [0-9]+ ('-' [0-9] [0-9]*)?)?) / ('#' '~'? '(' [0-9] WS? ('<' '<') WS? [0-9] [0-9]? ')') / (('#' / '$') '~'? ('0' 'x')? ([0-9] / [0-9] / ([a-f] / [A-F]))+) / ('$' '(' '-' [0-9]+ ')') / ARMRegister) !('f' / 'b' / ':' / '(' / '+' / '-'))> */ func() bool { + if memoized, ok := memoization[memoKey{39, position}]; ok { + return memoizedResult(memoized) + } position536, tokenIndex536 := position, tokenIndex { position537 := position @@ -4900,13 +5146,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleRegisterOrConstant, position537) } + memoize(39, position536, tokenIndex536, true) return true l536: + memoize(39, position536, tokenIndex536, false) position, tokenIndex = position536, tokenIndex536 return false }, /* 40 ARMConstantTweak <- <(((('u' / 's') (('x' / 'X') ('t' / 'T')) ('x' / 'w' / 'h' / 'b')) / (('l' / 'L') ('s' / 'S') ('l' / 'L')) / (('l' / 'L') ('s' / 'S') ('r' / 'R')) / (('r' / 'R') ('o' / 'O') ('r' / 'R')) / (('a' / 'A') ('s' / 'S') ('r' / 'R'))) (WS '#'? Offset)?)> */ func() bool { + if memoized, ok := memoization[memoKey{40, position}]; ok { + return memoizedResult(memoized) + } position616, tokenIndex616 := position, tokenIndex { position617 := position @@ -5206,13 +5457,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l658: add(ruleARMConstantTweak, position617) } + memoize(40, position616, tokenIndex616, true) return true l616: + memoize(40, position616, tokenIndex616, false) position, tokenIndex = position616, tokenIndex616 return false }, /* 41 ARMRegister <- <((('s' / 'S') ('p' / 'P')) / (('x' / 'w' / 'd' / 'q' / 's' / 'h' / 'b') [0-9] [0-9]?) / (('x' / 'X') ('z' / 'Z') ('r' / 'R')) / (('w' / 'W') ('z' / 'Z') ('r' / 'R')) / (('n' / 'N') ('z' / 'Z') ('c' / 'C') ('v' / 'V')) / ARMVectorRegister / ('{' WS? ARMVectorRegister WS? ((',' / '-') WS? ARMVectorRegister)* WS? '}' ('[' [0-9] [0-9]? ']')?))> */ func() bool { + if memoized, ok := memoization[memoKey{41, position}]; ok { + return memoizedResult(memoized) + } position661, tokenIndex661 := position, tokenIndex { position662 := position @@ -5594,13 +5850,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l663: add(ruleARMRegister, position662) } + memoize(41, position661, tokenIndex661, true) return true l661: + memoize(41, position661, tokenIndex661, false) position, tokenIndex = position661, tokenIndex661 return false }, /* 42 ARMVectorRegister <- <(('v' / 'V') [0-9] [0-9]? ('.' [0-9]* ('b' / 's' / 'd' / 'h' / 'q') ('[' [0-9] [0-9]? ']')?)?)> */ func() bool { + if memoized, ok := memoization[memoKey{42, position}]; ok { + return memoizedResult(memoized) + } position719, tokenIndex719 := position, tokenIndex { position720 := position @@ -5724,13 +5985,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l726: add(ruleARMVectorRegister, position720) } + memoize(42, position719, tokenIndex719, true) return true l719: + memoize(42, position719, tokenIndex719, false) position, tokenIndex = position719, tokenIndex719 return false }, /* 43 MemoryRef <- <((SymbolRef BaseIndexScale) / SymbolRef / Low12BitsSymbolRef / (Offset* BaseIndexScale) / (SegmentRegister Offset BaseIndexScale) / (SegmentRegister BaseIndexScale) / (SegmentRegister Offset) / ARMBaseIndexScale / BaseIndexScale)> */ func() bool { + if memoized, ok := memoization[memoKey{43, position}]; ok { + return memoizedResult(memoized) + } position738, tokenIndex738 := position, tokenIndex { position739 := position @@ -5816,13 +6082,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l740: add(ruleMemoryRef, position739) } + memoize(43, position738, tokenIndex738, true) return true l738: + memoize(43, position738, tokenIndex738, false) position, tokenIndex = position738, tokenIndex738 return false }, /* 44 SymbolRef <- <((Offset* '+')? (LocalSymbol / SymbolName) Offset* ('@' Section Offset*)?)> */ func() bool { + if memoized, ok := memoization[memoKey{44, position}]; ok { + return memoizedResult(memoized) + } position751, tokenIndex751 := position, tokenIndex { position752 := position @@ -5896,13 +6167,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l762: add(ruleSymbolRef, position752) } + memoize(44, position751, tokenIndex751, true) return true l751: + memoize(44, position751, tokenIndex751, false) position, tokenIndex = position751, tokenIndex751 return false }, /* 45 Low12BitsSymbolRef <- <(':' ('l' / 'L') ('o' / 'O') '1' '2' ':' (LocalSymbol / SymbolName) Offset?)> */ func() bool { + if memoized, ok := memoization[memoKey{45, position}]; ok { + return memoizedResult(memoized) + } position765, tokenIndex765 := position, tokenIndex { position766 := position @@ -5977,13 +6253,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l774: add(ruleLow12BitsSymbolRef, position766) } + memoize(45, position765, tokenIndex765, true) return true l765: + memoize(45, position765, tokenIndex765, false) position, tokenIndex = position765, tokenIndex765 return false }, /* 46 ARMBaseIndexScale <- <('[' ARMRegister (',' WS? (('#'? Offset (('*' [0-9]+) / ('*' '(' [0-9]+ Operator [0-9]+ ')') / ('+' [0-9]+)*)?) / ('#'? ARMGOTLow12) / ('#'? Low12BitsSymbolRef) / ARMRegister) (',' WS? ARMConstantTweak)?)? ']' ARMPostincrement?)> */ func() bool { + if memoized, ok := memoization[memoKey{46, position}]; ok { + return memoizedResult(memoized) + } position775, tokenIndex775 := position, tokenIndex { position776 := position @@ -6220,13 +6501,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l811: add(ruleARMBaseIndexScale, position776) } + memoize(46, position775, tokenIndex775, true) return true l775: + memoize(46, position775, tokenIndex775, false) position, tokenIndex = position775, tokenIndex775 return false }, /* 47 ARMGOTLow12 <- <(':' ('g' / 'G') ('o' / 'O') ('t' / 'T') '_' ('l' / 'L') ('o' / 'O') '1' '2' ':' SymbolName)> */ func() bool { + if memoized, ok := memoization[memoKey{47, position}]; ok { + return memoizedResult(memoized) + } position812, tokenIndex812 := position, tokenIndex { position813 := position @@ -6330,13 +6616,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } add(ruleARMGOTLow12, position813) } + memoize(47, position812, tokenIndex812, true) return true l812: + memoize(47, position812, tokenIndex812, false) position, tokenIndex = position812, tokenIndex812 return false }, /* 48 ARMPostincrement <- <'!'> */ func() bool { + if memoized, ok := memoization[memoKey{48, position}]; ok { + return memoizedResult(memoized) + } position824, tokenIndex824 := position, tokenIndex { position825 := position @@ -6346,13 +6637,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { position++ add(ruleARMPostincrement, position825) } + memoize(48, position824, tokenIndex824, true) return true l824: + memoize(48, position824, tokenIndex824, false) position, tokenIndex = position824, tokenIndex824 return false }, /* 49 BaseIndexScale <- <('(' RegisterOrConstant? WS? (',' WS? RegisterOrConstant WS? (',' [0-9]+)?)? ')')> */ func() bool { + if memoized, ok := memoization[memoKey{49, position}]; ok { + return memoizedResult(memoized) + } position826, tokenIndex826 := position, tokenIndex { position827 := position @@ -6446,13 +6742,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { position++ add(ruleBaseIndexScale, position827) } + memoize(49, position826, tokenIndex826, true) return true l826: + memoize(49, position826, tokenIndex826, false) position, tokenIndex = position826, tokenIndex826 return false }, /* 50 Operator <- <('+' / '-')> */ func() bool { + if memoized, ok := memoization[memoKey{50, position}]; ok { + return memoizedResult(memoized) + } position842, tokenIndex842 := position, tokenIndex { position843 := position @@ -6473,13 +6774,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l844: add(ruleOperator, position843) } + memoize(50, position842, tokenIndex842, true) return true l842: + memoize(50, position842, tokenIndex842, false) position, tokenIndex = position842, tokenIndex842 return false }, /* 51 OffsetOperator <- <('+' / '-' / '*')> */ func() bool { + if memoized, ok := memoization[memoKey{51, position}]; ok { + return memoizedResult(memoized) + } position846, tokenIndex846 := position, tokenIndex { position847 := position @@ -6507,13 +6813,18 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l848: add(ruleOffsetOperator, position847) } + memoize(51, position846, tokenIndex846, true) return true l846: + memoize(51, position846, tokenIndex846, false) position, tokenIndex = position846, tokenIndex846 return false }, - /* 52 Offset <- <('+'? '-'? (('0' ('b' / 'B') ('0' / '1')+) / ('0' ('x' / 'X') ([0-9] / [0-9] / ([a-f] / [A-F]))+) / ((([0-9]+ (OffsetOperator '(' [0-9]+ OffsetOperator [0-9]+ ')')?) / ([0-9]+ (OffsetOperator [0-9]+ OffsetOperator [0-9]+)?) / ([0-9]+ (OffsetOperator [0-9]+)?) / ('(' [0-9]+ WS? OffsetOperator WS? [0-9]+ ')')) !([a-z] / [A-Z]))))> */ + /* 52 Offset <- <('+'? '-'? (('0' ('b' / 'B') ('0' / '1')+) / ('0' ('x' / 'X') ([0-9] / [0-9] / ([a-f] / [A-F]))+) / ((([0-9]+ WS OffsetOperator [0-9]+) / ([0-9]+ (OffsetOperator '(' [0-9]+ OffsetOperator [0-9]+ ')')?) / ([0-9]+ (OffsetOperator [0-9]+ OffsetOperator [0-9]+)?) / ([0-9]+ (OffsetOperator [0-9]+)?) / ('(' [0-9]+ WS? OffsetOperator WS? [0-9]+ ')' OffsetOperator [0-9]+ OffsetOperator [0-9]+) / ('(' [0-9]+ WS? OffsetOperator WS? [0-9]+ ')') / ('(' [0-9]+ WS? OffsetOperator WS? [0-9]+ WS? OffsetOperator WS? [0-9]+ ')')) !([a-z] / [A-Z]))))> */ func() bool { + if memoized, ok := memoization[memoKey{52, position}]; ok { + return memoizedResult(memoized) + } position851, tokenIndex851 := position, tokenIndex { position852 := position @@ -6712,163 +7023,366 @@ func (p *Asm) Init(options ...func(*Asm) error) error { l885: position, tokenIndex = position885, tokenIndex885 } + if !_rules[ruleWS]() { + goto l883 + } + if !_rules[ruleOffsetOperator]() { + goto l883 + } + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l883 + } + position++ + l886: + { + position887, tokenIndex887 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l887 + } + position++ + goto l886 + l887: + position, tokenIndex = position887, tokenIndex887 + } + goto l882 + l883: + position, tokenIndex = position882, tokenIndex882 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l888 + } + position++ + l889: + { + position890, tokenIndex890 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l890 + } + position++ + goto l889 + l890: + position, tokenIndex = position890, tokenIndex890 + } { - position886, tokenIndex886 := position, tokenIndex + position891, tokenIndex891 := position, tokenIndex if !_rules[ruleOffsetOperator]() { - goto l886 + goto l891 } if buffer[position] != rune('(') { - goto l886 + goto l891 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l886 + goto l891 } position++ - l888: + l893: { - position889, tokenIndex889 := position, tokenIndex + position894, tokenIndex894 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l889 + goto l894 } position++ - goto l888 - l889: - position, tokenIndex = position889, tokenIndex889 + goto l893 + l894: + position, tokenIndex = position894, tokenIndex894 } if !_rules[ruleOffsetOperator]() { - goto l886 + goto l891 } if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l886 + goto l891 } position++ - l890: + l895: { - position891, tokenIndex891 := position, tokenIndex + position896, tokenIndex896 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l891 + goto l896 } position++ - goto l890 - l891: - position, tokenIndex = position891, tokenIndex891 + goto l895 + l896: + position, tokenIndex = position896, tokenIndex896 } if buffer[position] != rune(')') { - goto l886 + goto l891 } position++ - goto l887 - l886: - position, tokenIndex = position886, tokenIndex886 + goto l892 + l891: + position, tokenIndex = position891, tokenIndex891 } - l887: + l892: goto l882 - l883: + l888: position, tokenIndex = position882, tokenIndex882 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l892 + goto l897 } position++ - l893: + l898: { - position894, tokenIndex894 := position, tokenIndex + position899, tokenIndex899 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l894 + goto l899 } position++ - goto l893 - l894: - position, tokenIndex = position894, tokenIndex894 + goto l898 + l899: + position, tokenIndex = position899, tokenIndex899 } { - position895, tokenIndex895 := position, tokenIndex + position900, tokenIndex900 := position, tokenIndex if !_rules[ruleOffsetOperator]() { - goto l895 + goto l900 } if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l895 + goto l900 } position++ - l897: + l902: { - position898, tokenIndex898 := position, tokenIndex + position903, tokenIndex903 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l898 + goto l903 } position++ - goto l897 - l898: - position, tokenIndex = position898, tokenIndex898 + goto l902 + l903: + position, tokenIndex = position903, tokenIndex903 } if !_rules[ruleOffsetOperator]() { - goto l895 + goto l900 } if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l895 + goto l900 } position++ - l899: + l904: { - position900, tokenIndex900 := position, tokenIndex + position905, tokenIndex905 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l900 + goto l905 } position++ - goto l899 - l900: - position, tokenIndex = position900, tokenIndex900 + goto l904 + l905: + position, tokenIndex = position905, tokenIndex905 } - goto l896 - l895: - position, tokenIndex = position895, tokenIndex895 + goto l901 + l900: + position, tokenIndex = position900, tokenIndex900 } - l896: + l901: goto l882 - l892: + l897: position, tokenIndex = position882, tokenIndex882 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l901 + goto l906 } position++ - l902: + l907: { - position903, tokenIndex903 := position, tokenIndex + position908, tokenIndex908 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l903 + goto l908 } position++ - goto l902 - l903: - position, tokenIndex = position903, tokenIndex903 + goto l907 + l908: + position, tokenIndex = position908, tokenIndex908 } { - position904, tokenIndex904 := position, tokenIndex + position909, tokenIndex909 := position, tokenIndex if !_rules[ruleOffsetOperator]() { - goto l904 + goto l909 } if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l904 + goto l909 } position++ - l906: + l911: { - position907, tokenIndex907 := position, tokenIndex + position912, tokenIndex912 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l907 + goto l912 } position++ - goto l906 - l907: - position, tokenIndex = position907, tokenIndex907 + goto l911 + l912: + position, tokenIndex = position912, tokenIndex912 } - goto l905 - l904: - position, tokenIndex = position904, tokenIndex904 + goto l910 + l909: + position, tokenIndex = position909, tokenIndex909 } - l905: + l910: goto l882 - l901: + l906: + position, tokenIndex = position882, tokenIndex882 + if buffer[position] != rune('(') { + goto l913 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l913 + } + position++ + l914: + { + position915, tokenIndex915 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l915 + } + position++ + goto l914 + l915: + position, tokenIndex = position915, tokenIndex915 + } + { + position916, tokenIndex916 := position, tokenIndex + if !_rules[ruleWS]() { + goto l916 + } + goto l917 + l916: + position, tokenIndex = position916, tokenIndex916 + } + l917: + if !_rules[ruleOffsetOperator]() { + goto l913 + } + { + position918, tokenIndex918 := position, tokenIndex + if !_rules[ruleWS]() { + goto l918 + } + goto l919 + l918: + position, tokenIndex = position918, tokenIndex918 + } + l919: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l913 + } + position++ + l920: + { + position921, tokenIndex921 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l921 + } + position++ + goto l920 + l921: + position, tokenIndex = position921, tokenIndex921 + } + if buffer[position] != rune(')') { + goto l913 + } + position++ + if !_rules[ruleOffsetOperator]() { + goto l913 + } + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l913 + } + position++ + l922: + { + position923, tokenIndex923 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l923 + } + position++ + goto l922 + l923: + position, tokenIndex = position923, tokenIndex923 + } + if !_rules[ruleOffsetOperator]() { + goto l913 + } + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l913 + } + position++ + l924: + { + position925, tokenIndex925 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l925 + } + position++ + goto l924 + l925: + position, tokenIndex = position925, tokenIndex925 + } + goto l882 + l913: + position, tokenIndex = position882, tokenIndex882 + if buffer[position] != rune('(') { + goto l926 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l926 + } + position++ + l927: + { + position928, tokenIndex928 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l928 + } + position++ + goto l927 + l928: + position, tokenIndex = position928, tokenIndex928 + } + { + position929, tokenIndex929 := position, tokenIndex + if !_rules[ruleWS]() { + goto l929 + } + goto l930 + l929: + position, tokenIndex = position929, tokenIndex929 + } + l930: + if !_rules[ruleOffsetOperator]() { + goto l926 + } + { + position931, tokenIndex931 := position, tokenIndex + if !_rules[ruleWS]() { + goto l931 + } + goto l932 + l931: + position, tokenIndex = position931, tokenIndex931 + } + l932: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l926 + } + position++ + l933: + { + position934, tokenIndex934 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l934 + } + position++ + goto l933 + l934: + position, tokenIndex = position934, tokenIndex934 + } + if buffer[position] != rune(')') { + goto l926 + } + position++ + goto l882 + l926: position, tokenIndex = position882, tokenIndex882 if buffer[position] != rune('(') { goto l851 @@ -6878,54 +7392,92 @@ func (p *Asm) Init(options ...func(*Asm) error) error { goto l851 } position++ - l908: + l935: { - position909, tokenIndex909 := position, tokenIndex + position936, tokenIndex936 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l909 + goto l936 } position++ - goto l908 - l909: - position, tokenIndex = position909, tokenIndex909 + goto l935 + l936: + position, tokenIndex = position936, tokenIndex936 } { - position910, tokenIndex910 := position, tokenIndex + position937, tokenIndex937 := position, tokenIndex if !_rules[ruleWS]() { - goto l910 + goto l937 } - goto l911 - l910: - position, tokenIndex = position910, tokenIndex910 + goto l938 + l937: + position, tokenIndex = position937, tokenIndex937 } - l911: + l938: if !_rules[ruleOffsetOperator]() { goto l851 } { - position912, tokenIndex912 := position, tokenIndex + position939, tokenIndex939 := position, tokenIndex if !_rules[ruleWS]() { - goto l912 + goto l939 } - goto l913 - l912: - position, tokenIndex = position912, tokenIndex912 + goto l940 + l939: + position, tokenIndex = position939, tokenIndex939 } - l913: + l940: if c := buffer[position]; c < rune('0') || c > rune('9') { goto l851 } position++ - l914: + l941: { - position915, tokenIndex915 := position, tokenIndex + position942, tokenIndex942 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l915 + goto l942 } position++ - goto l914 - l915: - position, tokenIndex = position915, tokenIndex915 + goto l941 + l942: + position, tokenIndex = position942, tokenIndex942 + } + { + position943, tokenIndex943 := position, tokenIndex + if !_rules[ruleWS]() { + goto l943 + } + goto l944 + l943: + position, tokenIndex = position943, tokenIndex943 + } + l944: + if !_rules[ruleOffsetOperator]() { + goto l851 + } + { + position945, tokenIndex945 := position, tokenIndex + if !_rules[ruleWS]() { + goto l945 + } + goto l946 + l945: + position, tokenIndex = position945, tokenIndex945 + } + l946: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l851 + } + position++ + l947: + { + position948, tokenIndex948 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l948 + } + position++ + goto l947 + l948: + position, tokenIndex = position948, tokenIndex948 } if buffer[position] != rune(')') { goto l851 @@ -6934,135 +7486,147 @@ func (p *Asm) Init(options ...func(*Asm) error) error { } l882: { - position916, tokenIndex916 := position, tokenIndex + position949, tokenIndex949 := position, tokenIndex { - position917, tokenIndex917 := position, tokenIndex + position950, tokenIndex950 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l918 + goto l951 } position++ - goto l917 - l918: - position, tokenIndex = position917, tokenIndex917 + goto l950 + l951: + position, tokenIndex = position950, tokenIndex950 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l916 + goto l949 } position++ } - l917: + l950: goto l851 - l916: - position, tokenIndex = position916, tokenIndex916 + l949: + position, tokenIndex = position949, tokenIndex949 } } l857: add(ruleOffset, position852) } + memoize(52, position851, tokenIndex851, true) return true l851: + memoize(52, position851, tokenIndex851, false) position, tokenIndex = position851, tokenIndex851 return false }, /* 53 Section <- <([a-z] / [A-Z] / '@')+> */ func() bool { - position919, tokenIndex919 := position, tokenIndex + if memoized, ok := memoization[memoKey{53, position}]; ok { + return memoizedResult(memoized) + } + position952, tokenIndex952 := position, tokenIndex { - position920 := position + position953 := position { - position923, tokenIndex923 := position, tokenIndex + position956, tokenIndex956 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l924 + goto l957 } position++ - goto l923 - l924: - position, tokenIndex = position923, tokenIndex923 + goto l956 + l957: + position, tokenIndex = position956, tokenIndex956 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l925 + goto l958 } position++ - goto l923 - l925: - position, tokenIndex = position923, tokenIndex923 + goto l956 + l958: + position, tokenIndex = position956, tokenIndex956 if buffer[position] != rune('@') { - goto l919 + goto l952 } position++ } - l923: - l921: + l956: + l954: { - position922, tokenIndex922 := position, tokenIndex + position955, tokenIndex955 := position, tokenIndex { - position926, tokenIndex926 := position, tokenIndex + position959, tokenIndex959 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l927 + goto l960 } position++ - goto l926 - l927: - position, tokenIndex = position926, tokenIndex926 + goto l959 + l960: + position, tokenIndex = position959, tokenIndex959 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l928 + goto l961 } position++ - goto l926 - l928: - position, tokenIndex = position926, tokenIndex926 + goto l959 + l961: + position, tokenIndex = position959, tokenIndex959 if buffer[position] != rune('@') { - goto l922 + goto l955 } position++ } - l926: - goto l921 - l922: - position, tokenIndex = position922, tokenIndex922 + l959: + goto l954 + l955: + position, tokenIndex = position955, tokenIndex955 } - add(ruleSection, position920) + add(ruleSection, position953) } + memoize(53, position952, tokenIndex952, true) return true - l919: - position, tokenIndex = position919, tokenIndex919 + l952: + memoize(53, position952, tokenIndex952, false) + position, tokenIndex = position952, tokenIndex952 return false }, /* 54 SegmentRegister <- <('%' ([c-g] / 's') ('s' ':'))> */ func() bool { - position929, tokenIndex929 := position, tokenIndex + if memoized, ok := memoization[memoKey{54, position}]; ok { + return memoizedResult(memoized) + } + position962, tokenIndex962 := position, tokenIndex { - position930 := position + position963 := position if buffer[position] != rune('%') { - goto l929 + goto l962 } position++ { - position931, tokenIndex931 := position, tokenIndex + position964, tokenIndex964 := position, tokenIndex if c := buffer[position]; c < rune('c') || c > rune('g') { - goto l932 + goto l965 } position++ - goto l931 - l932: - position, tokenIndex = position931, tokenIndex931 + goto l964 + l965: + position, tokenIndex = position964, tokenIndex964 if buffer[position] != rune('s') { - goto l929 + goto l962 } position++ } - l931: + l964: if buffer[position] != rune('s') { - goto l929 + goto l962 } position++ if buffer[position] != rune(':') { - goto l929 + goto l962 } position++ - add(ruleSegmentRegister, position930) + add(ruleSegmentRegister, position963) } + memoize(54, position962, tokenIndex962, true) return true - l929: - position, tokenIndex = position929, tokenIndex929 + l962: + memoize(54, position962, tokenIndex962, false) + position, tokenIndex = position962, tokenIndex962 return false }, }