From 72e22ed0edfb0c29d3cc68fef5d53ea338bbae28 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 19 Oct 2026 23:22:23 -0500 Subject: [PATCH] Run SHA-384 and SHA-512 blocks with AVX2, AVX-512VL or the arm64 SHA-512 instructions SHA-384, SHA-512, SHA-512/224 and SHA-512/256 (createHash, createHmac, crypto.hash, pbkdf2, hkdf) process their blocks with the fastest kernel the CPU supports: - x86-64 with AVX2 and BMI2: the message schedule is computed two words at a time in vector registers, interleaved with the scalar rounds, which rotate with RORX. - x86-64 with AVX-512VL: the same, with vector rotates and three-input XOR in the schedule. - arm64 with the SHA-512 extension: SHA512H, SHA512H2, SHA512SU0 and SHA512SU1. - Otherwise the portable blocks, now unrolled over a 16-word schedule ring. The CPU and OS checks run once and are cached. The update and final paths pass all their whole blocks to the kernel in one call. test_sha_kernels.c checks the FIPS 180-4 SHA-384 and SHA-512 vectors through every kernel the CPU supports and compares each kernel with the portable blocks over every length from 0 to 1100 bytes and 1 MiB, whole and in 1-, 7-, 64- and 100-byte updates. Corpus 5779 compares SHA-384, SHA-512 and HMAC-SHA512 digests over many lengths and update sizes with Node; tests/harness/corpus-costs/ records its plain and sanitized costs. --- .../tests/corpus/5779-sha512-lengths.ts.json | 6 + packages/runtime/src/scr_lib.c | 344 ++++++++++++++++-- packages/runtime/test/sha-kernels.test.ts | 4 +- packages/runtime/test/test_sha_kernels.c | 100 ++++- tests/corpus/5779-sha512-lengths.ts | 35 ++ .../corpus-costs/5779-sha512-lengths.ts.json | 1 + 6 files changed, 456 insertions(+), 34 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5779-sha512-lengths.ts.json create mode 100644 tests/corpus/5779-sha512-lengths.ts create mode 100644 tests/harness/corpus-costs/5779-sha512-lengths.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5779-sha512-lengths.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5779-sha512-lengths.ts.json new file mode 100644 index 000000000..60582ec60 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5779-sha512-lengths.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5779-sha512-lengths.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index 930d44818..99c5a8f63 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -7405,29 +7405,319 @@ static uint64_t scr_sha512_rotr(uint64_t x, unsigned n) { return (x >> n) | (x << (64 - n)); } -SCR_DIGEST_ALIGNED static void scr_sha512_block(uint64_t h[8], const unsigned char *data) { - uint64_t w[80]; - for (size_t i = 0; i < 16; i++) { - w[i] = 0; - for (size_t b = 0; b < 8; b++) w[i] = (w[i] << 8) | data[i * 8 + b]; +#define SCR_SHA512_BSIG0(x) \ + (scr_sha512_rotr(x, 28) ^ scr_sha512_rotr(x, 34) ^ scr_sha512_rotr(x, 39)) +#define SCR_SHA512_BSIG1(x) \ + (scr_sha512_rotr(x, 14) ^ scr_sha512_rotr(x, 18) ^ scr_sha512_rotr(x, 41)) +#define SCR_SHA512_SSIG0(x) (scr_sha512_rotr(x, 1) ^ scr_sha512_rotr(x, 8) ^ ((x) >> 7)) +#define SCR_SHA512_SSIG1(x) (scr_sha512_rotr(x, 19) ^ scr_sha512_rotr(x, 61) ^ ((x) >> 6)) + +/* One round with the eight working variables passed in rotated order, so + * an unrolled sequence of rounds needs no register moves. */ +#define SCR_SHA512_ROUND(a, b, c, d, e, f, g, h, k, w) \ + do { \ + uint64_t t1 = (h) + SCR_SHA512_BSIG1(e) + ((g) ^ ((e) & ((f) ^ (g)))) + (k) + (w); \ + (d) += t1; \ + (h) = t1 + SCR_SHA512_BSIG0(a) + (((a) & (b)) | ((c) & ((a) | (b)))); \ + } while (0) + +static uint64_t scr_sha512_load(const unsigned char *p) { + uint64_t v = 0; + for (size_t b = 0; b < 8; b++) v = (v << 8) | p[b]; + return v; +} + +/* The message schedule is kept as a 16-word ring and extended 16 words + * at a time between groups of 16 rounds. */ +SCR_DIGEST_ALIGNED static void scr_sha512_blocks_portable(uint64_t state[8], const unsigned char *p, + size_t blocks) { + uint64_t a = state[0], b = state[1], c = state[2], d = state[3]; + uint64_t e = state[4], f = state[5], g = state[6], h = state[7]; + for (; blocks != 0; blocks--, p += 128) { + uint64_t w[16]; + for (size_t i = 0; i < 16; i++) w[i] = scr_sha512_load(p + i * 8); + for (size_t i = 0; i < 80; i += 16) { + if (i != 0) { + for (size_t j = 0; j < 16; j++) { + w[j] += SCR_SHA512_SSIG1(w[(j + 14) & 15]) + w[(j + 9) & 15] + + SCR_SHA512_SSIG0(w[(j + 1) & 15]); + } + } + const uint64_t *k = scr_sha512_k + i; + SCR_SHA512_ROUND(a, b, c, d, e, f, g, h, k[0], w[0]); + SCR_SHA512_ROUND(h, a, b, c, d, e, f, g, k[1], w[1]); + SCR_SHA512_ROUND(g, h, a, b, c, d, e, f, k[2], w[2]); + SCR_SHA512_ROUND(f, g, h, a, b, c, d, e, k[3], w[3]); + SCR_SHA512_ROUND(e, f, g, h, a, b, c, d, k[4], w[4]); + SCR_SHA512_ROUND(d, e, f, g, h, a, b, c, k[5], w[5]); + SCR_SHA512_ROUND(c, d, e, f, g, h, a, b, k[6], w[6]); + SCR_SHA512_ROUND(b, c, d, e, f, g, h, a, k[7], w[7]); + SCR_SHA512_ROUND(a, b, c, d, e, f, g, h, k[8], w[8]); + SCR_SHA512_ROUND(h, a, b, c, d, e, f, g, k[9], w[9]); + SCR_SHA512_ROUND(g, h, a, b, c, d, e, f, k[10], w[10]); + SCR_SHA512_ROUND(f, g, h, a, b, c, d, e, k[11], w[11]); + SCR_SHA512_ROUND(e, f, g, h, a, b, c, d, k[12], w[12]); + SCR_SHA512_ROUND(d, e, f, g, h, a, b, c, k[13], w[13]); + SCR_SHA512_ROUND(c, d, e, f, g, h, a, b, k[14], w[14]); + SCR_SHA512_ROUND(b, c, d, e, f, g, h, a, k[15], w[15]); + } + a = state[0] += a; b = state[1] += b; c = state[2] += c; d = state[3] += d; + e = state[4] += e; f = state[5] += f; g = state[6] += g; h = state[7] += h; } - for (size_t i = 16; i < 80; i++) { - uint64_t s0 = scr_sha512_rotr(w[i - 15], 1) ^ scr_sha512_rotr(w[i - 15], 8) ^ (w[i - 15] >> 7); - uint64_t s1 = scr_sha512_rotr(w[i - 2], 19) ^ scr_sha512_rotr(w[i - 2], 61) ^ (w[i - 2] >> 6); - w[i] = w[i - 16] + s0 + w[i - 7] + s1; +} + +#ifdef SCR_SHA_NI +/* ── SHA-512 kernels (x86-64) ───────────────────────────────────────── + * With AVX2 and BMI2, the message schedule is computed two words at a + * time in vector registers, interleaved with the scalar rounds, and the + * rounds rotate with RORX. With AVX-512VL the schedule uses vector + * rotates and three-input XOR. The CPU and OS state checks run once and + * are cached. */ +static int scr_sha512_x86_state; /* 0 unknown, 1 portable, 2 AVX2, 3 AVX-512VL */ + +static int scr_sha512_x86(void) { + if (scr_sha512_x86_state == 0) { + unsigned a, b, c, d; + int kind = 1; + if (__get_cpuid(1, &a, &b, &c, &d) && (c & bit_OSXSAVE) && (c & bit_AVX)) { + unsigned lo, hi; + __asm__ volatile("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0)); + (void)hi; + /* XMM and YMM state enabled. */ + if ((lo & 6) == 6 && __get_cpuid_count(7, 0, &a, &b, &c, &d) && (b & bit_AVX2) && + (b & bit_BMI2)) { + kind = 2; + /* Opmask, ZMM0-15 upper halves and ZMM16-31 state enabled. */ + if ((b & bit_AVX512F) && (b & bit_AVX512VL) && (lo & 0xe0) == 0xe0) kind = 3; + } + } + scr_sha512_x86_state = kind; } - uint64_t a = h[0], b = h[1], c = h[2], d = h[3]; - uint64_t e = h[4], f = h[5], g = h[6], v = h[7]; - for (size_t i = 0; i < 80; i++) { - uint64_t s1 = scr_sha512_rotr(e, 14) ^ scr_sha512_rotr(e, 18) ^ scr_sha512_rotr(e, 41); - uint64_t t1 = v + s1 + ((e & f) ^ (~e & g)) + scr_sha512_k[i] + w[i]; - uint64_t s0 = scr_sha512_rotr(a, 28) ^ scr_sha512_rotr(a, 34) ^ scr_sha512_rotr(a, 39); - uint64_t t2 = s0 + ((a & b) ^ (a & c) ^ (b & c)); - v = g; g = f; f = e; e = d + t1; - d = c; c = b; b = a; a = t1 + t2; + return scr_sha512_x86_state; +} + +__attribute__((target("avx2,bmi2"))) static inline __m128i scr_sha512_ssig0_avx2(__m128i x) { + return _mm_xor_si128(_mm_xor_si128(_mm_or_si128(_mm_srli_epi64(x, 1), _mm_slli_epi64(x, 63)), + _mm_or_si128(_mm_srli_epi64(x, 8), _mm_slli_epi64(x, 56))), + _mm_srli_epi64(x, 7)); +} + +__attribute__((target("avx2,bmi2"))) static inline __m128i scr_sha512_ssig1_avx2(__m128i x) { + return _mm_xor_si128(_mm_xor_si128(_mm_or_si128(_mm_srli_epi64(x, 19), _mm_slli_epi64(x, 45)), + _mm_or_si128(_mm_srli_epi64(x, 61), _mm_slli_epi64(x, 3))), + _mm_srli_epi64(x, 6)); +} + +__attribute__((target("avx512f,avx512vl,bmi2"))) static inline __m128i +scr_sha512_ssig0_avx512(__m128i x) { + return _mm_ternarylogic_epi64(_mm_ror_epi64(x, 1), _mm_ror_epi64(x, 8), _mm_srli_epi64(x, 7), + 0x96); +} + +__attribute__((target("avx512f,avx512vl,bmi2"))) static inline __m128i +scr_sha512_ssig1_avx512(__m128i x) { + return _mm_ternarylogic_epi64(_mm_ror_epi64(x, 19), _mm_ror_epi64(x, 61), _mm_srli_epi64(x, 6), + 0x96); +} + +/* A round for the vector kernels. wk is the schedule word plus its + * constant. Maj(a, b, c) is ((a ^ b) & (b ^ c)) ^ b, where b ^ c is the + * previous round's a ^ b. The empty asm statements keep the compiler from + * reassociating the additions: adding Sigma1(e) last keeps the sum of h, + * wk and Ch off the dependency chain through e. */ +#define SCR_SHA512_PIN(v) __asm__("" : "+r"(v)) +#define SCR_SHA512_ROUND_WK(a, b, c, d, e, f, g, h, wk, ab, bc) \ + do { \ + uint64_t t1 = (h) + (wk); \ + SCR_SHA512_PIN(t1); \ + t1 += (g) ^ ((e) & ((f) ^ (g))); \ + SCR_SHA512_PIN(t1); \ + t1 += SCR_SHA512_BSIG1(e); \ + SCR_SHA512_PIN(t1); \ + (d) += t1; \ + (ab) = (a) ^ (b); \ + uint64_t m = t1 + (((ab) & (bc)) ^ (b)); \ + SCR_SHA512_PIN(m); \ + (h) = m + SCR_SHA512_BSIG0(a); \ + } while (0) + +/* Words 2j and 2j+1 of the schedule, in the ring slot of words 2j-16 and + * 2j-15, and their sum with the round constants into wk. */ +#define SCR_SHA512_SCHEDULE(j, ssig0, ssig1) \ + do { \ + if ((j) < 40) { \ + __m128i s15 = _mm_alignr_epi8(w[((j) + 1) & 7], w[(j) & 7], 8); \ + __m128i s7 = _mm_alignr_epi8(w[((j) + 5) & 7], w[((j) + 4) & 7], 8); \ + __m128i next = _mm_add_epi64(_mm_add_epi64(w[(j) & 7], ssig0(s15)), \ + _mm_add_epi64(s7, ssig1(w[((j) + 7) & 7]))); \ + w[(j) & 7] = next; \ + _mm_store_si128((__m128i *)&wk[(j) * 2], \ + _mm_add_epi64(next, _mm_loadu_si128((const __m128i *)&scr_sha512_k[(j) * 2]))); \ + } \ + } while (0) + +/* Rounds 2i to 2i+7 compute the schedule words 2i+16 to 2i+23. */ +#define SCR_SHA512_VECTOR_BODY(ssig0, ssig1) \ + const __m128i bswap = _mm_set_epi64x(0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL); \ + uint64_t a = state[0], b = state[1], c = state[2], d = state[3]; \ + uint64_t e = state[4], f = state[5], g = state[6], h = state[7]; \ + for (; blocks != 0; blocks--, p += 128) { \ + __m128i w[8]; \ + uint64_t wk[80] __attribute__((aligned(16))); \ + _Pragma("GCC unroll 8") for (int i = 0; i < 8; i++) { \ + w[i] = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(p + i * 16)), bswap); \ + _mm_store_si128((__m128i *)&wk[i * 2], \ + _mm_add_epi64(w[i], _mm_loadu_si128((const __m128i *)&scr_sha512_k[i * 2]))); \ + } \ + uint64_t x = b ^ c, y; \ + _Pragma("GCC unroll 10") for (int i = 0; i < 40; i += 4) { \ + SCR_SHA512_SCHEDULE(i + 8, ssig0, ssig1); \ + SCR_SHA512_ROUND_WK(a, b, c, d, e, f, g, h, wk[2 * i + 0], y, x); \ + SCR_SHA512_ROUND_WK(h, a, b, c, d, e, f, g, wk[2 * i + 1], x, y); \ + SCR_SHA512_SCHEDULE(i + 9, ssig0, ssig1); \ + SCR_SHA512_ROUND_WK(g, h, a, b, c, d, e, f, wk[2 * i + 2], y, x); \ + SCR_SHA512_ROUND_WK(f, g, h, a, b, c, d, e, wk[2 * i + 3], x, y); \ + SCR_SHA512_SCHEDULE(i + 10, ssig0, ssig1); \ + SCR_SHA512_ROUND_WK(e, f, g, h, a, b, c, d, wk[2 * i + 4], y, x); \ + SCR_SHA512_ROUND_WK(d, e, f, g, h, a, b, c, wk[2 * i + 5], x, y); \ + SCR_SHA512_SCHEDULE(i + 11, ssig0, ssig1); \ + SCR_SHA512_ROUND_WK(c, d, e, f, g, h, a, b, wk[2 * i + 6], y, x); \ + SCR_SHA512_ROUND_WK(b, c, d, e, f, g, h, a, wk[2 * i + 7], x, y); \ + } \ + a = state[0] += a; b = state[1] += b; c = state[2] += c; d = state[3] += d; \ + e = state[4] += e; f = state[5] += f; g = state[6] += g; h = state[7] += h; \ } - h[0] += a; h[1] += b; h[2] += c; h[3] += d; - h[4] += e; h[5] += f; h[6] += g; h[7] += v; + +__attribute__((target("avx2,bmi2"), aligned(64))) static void +scr_sha512_blocks_avx2(uint64_t state[8], const unsigned char *p, size_t blocks) { + SCR_SHA512_VECTOR_BODY(scr_sha512_ssig0_avx2, scr_sha512_ssig1_avx2) +} + +__attribute__((target("avx512f,avx512vl,bmi2"), aligned(64))) static void +scr_sha512_blocks_avx512(uint64_t state[8], const unsigned char *p, size_t blocks) { + SCR_SHA512_VECTOR_BODY(scr_sha512_ssig0_avx512, scr_sha512_ssig1_avx512) +} +#endif + + +#if defined(__aarch64__) && (defined(__clang__) || defined(__GNUC__)) && !defined(_WIN32) +/* ── SHA-512 kernel (arm64) ─────────────────────────────────────────── + * On CPUs with the SHA-512 extension (FEAT_SHA512), the blocks run with + * SHA512H, SHA512H2, SHA512SU0 and SHA512SU1; other CPUs use the portable + * blocks. The CPU check runs once and is cached. */ +#include +#if defined(__APPLE__) +#include +#else +#include +#endif +#define SCR_SHA512_ARM 1 +#if defined(__clang__) +#define SCR_SHA512_ARM_TARGET __attribute__((target("sha3"))) +#else +#define SCR_SHA512_ARM_TARGET __attribute__((target("+sha3"))) +#endif + +static int scr_sha512_arm_state; /* 0 unknown, 1 available, 2 absent */ + +static bool scr_sha512_arm(void) { + if (scr_sha512_arm_state == 0) { + bool ok; +#if defined(__APPLE__) + int value = 0; + size_t size = sizeof value; + ok = sysctlbyname("hw.optional.armv8_2_sha512", &value, &size, NULL, 0) == 0 && value != 0; +#else + ok = (getauxval(AT_HWCAP) & (1ul << 21)) != 0; /* HWCAP_SHA512 */ +#endif + scr_sha512_arm_state = ok ? 1 : 2; + } + return scr_sha512_arm_state == 1; +} + +/* Two rounds per step. With the state pairs in the roles (A, B, C, D) = + * (ab, cd, ef, gh), a step computes I = SHA512H(swap(K + W) + D, ext(C, D), + * ext(B, C)), D = SHA512H2(I, B, A) and B += I; the roles then rotate to + * (D, A, B, C). */ +#define SCR_SHA512_ARM_STEP(A, B, C, D, msg, k) \ + do { \ + uint64x2_t kw = vaddq_u64((msg), vld1q_u64(k)); \ + uint64x2_t sum = vaddq_u64(vextq_u64(kw, kw, 1), (D)); \ + uint64x2_t i = vsha512hq_u64(sum, vextq_u64((C), (D), 1), vextq_u64((B), (C), 1)); \ + (D) = vsha512h2q_u64(i, (B), (A)); \ + (B) = vaddq_u64((B), i); \ + } while (0) + +/* Message words 2j and 2j+1 from the eight pairs before them. */ +#define SCR_SHA512_ARM_SCHEDULE(m0, m1, m4, m5, m7) \ + (m0) = vsha512su1q_u64(vsha512su0q_u64((m0), (m1)), (m7), vextq_u64((m4), (m5), 1)) + +SCR_SHA512_ARM_TARGET static void scr_sha512_blocks_arm(uint64_t state[8], const unsigned char *p, + size_t blocks) { + uint64x2_t ab = vld1q_u64(&state[0]), cd = vld1q_u64(&state[2]); + uint64x2_t ef = vld1q_u64(&state[4]), gh = vld1q_u64(&state[6]); + for (; blocks != 0; blocks--, p += 128) { + uint64x2_t ab0 = ab, cd0 = cd, ef0 = ef, gh0 = gh; + uint64x2_t m0 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 0))); + uint64x2_t m1 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 16))); + uint64x2_t m2 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 32))); + uint64x2_t m3 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 48))); + uint64x2_t m4 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 64))); + uint64x2_t m5 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 80))); + uint64x2_t m6 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 96))); + uint64x2_t m7 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 112))); + const uint64_t *k = scr_sha512_k; + for (int i = 0; i < 5; i++, k += 16) { + SCR_SHA512_ARM_STEP(ab, cd, ef, gh, m0, k + 0); + SCR_SHA512_ARM_STEP(gh, ab, cd, ef, m1, k + 2); + SCR_SHA512_ARM_STEP(ef, gh, ab, cd, m2, k + 4); + SCR_SHA512_ARM_STEP(cd, ef, gh, ab, m3, k + 6); + SCR_SHA512_ARM_STEP(ab, cd, ef, gh, m4, k + 8); + SCR_SHA512_ARM_STEP(gh, ab, cd, ef, m5, k + 10); + SCR_SHA512_ARM_STEP(ef, gh, ab, cd, m6, k + 12); + SCR_SHA512_ARM_STEP(cd, ef, gh, ab, m7, k + 14); + if (i == 4) break; + SCR_SHA512_ARM_SCHEDULE(m0, m1, m4, m5, m7); + SCR_SHA512_ARM_SCHEDULE(m1, m2, m5, m6, m0); + SCR_SHA512_ARM_SCHEDULE(m2, m3, m6, m7, m1); + SCR_SHA512_ARM_SCHEDULE(m3, m4, m7, m0, m2); + SCR_SHA512_ARM_SCHEDULE(m4, m5, m0, m1, m3); + SCR_SHA512_ARM_SCHEDULE(m5, m6, m1, m2, m4); + SCR_SHA512_ARM_SCHEDULE(m6, m7, m2, m3, m5); + SCR_SHA512_ARM_SCHEDULE(m7, m0, m3, m4, m6); + } + ab = vaddq_u64(ab, ab0); + cd = vaddq_u64(cd, cd0); + ef = vaddq_u64(ef, ef0); + gh = vaddq_u64(gh, gh0); + } + vst1q_u64(&state[0], ab); + vst1q_u64(&state[2], cd); + vst1q_u64(&state[4], ef); + vst1q_u64(&state[6], gh); +} +#endif + +static void scr_sha512_blocks(uint64_t h[8], const unsigned char *p, size_t blocks) { +#ifdef SCR_SHA_NI + switch (scr_sha512_x86()) { + case 3: + scr_sha512_blocks_avx512(h, p, blocks); + return; + case 2: + scr_sha512_blocks_avx2(h, p, blocks); + return; + default: + break; + } +#endif +#ifdef SCR_SHA512_ARM + if (scr_sha512_arm()) { + scr_sha512_blocks_arm(h, p, blocks); + return; + } +#endif + scr_sha512_blocks_portable(h, p, blocks); } static void scr_sha512_init(ScrSha512Ctx *ctx, bool sha384) { @@ -7456,14 +7746,15 @@ static void scr_sha512_update(ScrSha512Ctx *ctx, const unsigned char *data, size data += take; len -= take; if (ctx->tail_len == 128) { - scr_sha512_block(ctx->h, ctx->tail); + scr_sha512_blocks(ctx->h, ctx->tail, 1); ctx->tail_len = 0; } } - while (len >= 128) { - scr_sha512_block(ctx->h, data); - data += 128; - len -= 128; + if (len >= 128) { + size_t blocks = len / 128; + scr_sha512_blocks(ctx->h, data, blocks); + data += blocks * 128; + len -= blocks * 128; } if (len != 0) { memcpy(ctx->tail, data, len); @@ -7482,8 +7773,7 @@ static size_t scr_sha512_final(const ScrSha512Ctx *source, unsigned char out[64] tail[pad - 1 - b] = (unsigned char)(low >> (8 * b)); tail[pad - 9 - b] = (unsigned char)(high >> (8 * b)); } - scr_sha512_block(ctx.h, tail); - if (pad == 256) scr_sha512_block(ctx.h, tail + 128); + scr_sha512_blocks(ctx.h, tail, pad / 128); size_t len = sha384 ? 48 : 64; for (size_t i = 0; i < len; i++) out[i] = (unsigned char)(ctx.h[i / 8] >> (56 - 8 * (i % 8))); return len; diff --git a/packages/runtime/test/sha-kernels.test.ts b/packages/runtime/test/sha-kernels.test.ts index 724841fb1..6f7759348 100644 --- a/packages/runtime/test/sha-kernels.test.ts +++ b/packages/runtime/test/sha-kernels.test.ts @@ -42,7 +42,7 @@ const runtimeSources = [ ]; // Compiled once with ASan, UBSan and the RC audit. The test runs the SHA-NI and portable -// SHA-1/SHA-256 blocks over the same inputs and checks known vectors. +// SHA-1/SHA-256 blocks and the SHA-512 kernels over the same inputs and checks known vectors. beforeAll(async () => { await mkdir(join(testDir, "build"), { recursive: true }); await execFileAsync("clang", [ @@ -64,7 +64,7 @@ beforeAll(async () => { ]); }, 300_000); -test("SHA-1 and SHA-256: the SHA-NI blocks match the portable blocks", async () => { +test("SHA-1, SHA-256, SHA-384 and SHA-512: the CPU kernels match the portable blocks", async () => { const { stderr } = await execFileAsync(bin, [], { env: { ...process.env, UBSAN_OPTIONS: "halt_on_error=1" }, }); diff --git a/packages/runtime/test/test_sha_kernels.c b/packages/runtime/test/test_sha_kernels.c index 7f854216e..f9f9b9df7 100644 --- a/packages/runtime/test/test_sha_kernels.c +++ b/packages/runtime/test/test_sha_kernels.c @@ -1,9 +1,11 @@ -/* SHA-1 and SHA-256 block kernels (scr_lib.c), built by +/* SHA-1, SHA-256, SHA-384 and SHA-512 block kernels (scr_lib.c), built by * sha-kernels.test.ts. The unit is included directly so the test can run - * the SHA-NI blocks and the portable blocks over the same inputs: known - * vectors, every length from 0 to 1100 bytes, and 1 MiB, each fed whole - * and in 1-, 7-, 64- and 100-byte updates. On CPUs without the SHA - * extensions only the portable blocks run. */ + * each kernel the CPU supports and the portable blocks over the same + * inputs: known vectors, every length from 0 to 1100 bytes, and 1 MiB, + * each fed whole and in 1-, 7-, 64- and 100-byte updates. The SHA-1 and + * SHA-256 kernels use the SHA extensions; the SHA-512 kernels use AVX2 or + * AVX-512VL on x86-64 and the SHA-512 extension on arm64. On CPUs without + * them only the portable blocks run. */ #include "../src/scr_lib.c" #include @@ -45,6 +47,71 @@ static void sha1_of(const unsigned char *data, size_t len, size_t chunk, unsigne scr_sha1_final(&ctx, out); } +static void sha512_of(const unsigned char *data, size_t len, size_t chunk, bool sha384, + unsigned char out[64]) { + ScrSha512Ctx ctx; + scr_sha512_init(&ctx, sha384); + for (size_t at = 0; at < len;) { + size_t take = chunk == 0 || chunk > len - at ? len - at : chunk; + scr_sha512_update(&ctx, data + at, take); + at += take; + } + scr_sha512_final(&ctx, out, sha384); +} + +/* FIPS 180-4 example vectors for SHA-384 and SHA-512. */ +static void vectors512(const char *path) { + unsigned char d[64]; + char text[129]; + static const char two_block[] = + "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; + sha512_of((const unsigned char *)"abc", 3, 0, false, d); + hex(d, 64, text); + check(strcmp(text, "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f") == 0, path, 3, 0); + sha512_of((const unsigned char *)"", 0, 0, false, d); + hex(d, 64, text); + check(strcmp(text, "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e") == 0, path, 0, 0); + sha512_of((const unsigned char *)two_block, sizeof two_block - 1, 0, false, d); + hex(d, 64, text); + check(strcmp(text, "8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa17299aeadb6889018501d289e4900f7e4331b99dec4b5433ac7d329eeb6dd26545e96e55b874be909") == 0, path, sizeof two_block - 1, 0); + sha512_of((const unsigned char *)"abc", 3, 0, true, d); + hex(d, 48, text); + check(strcmp(text, "cb00753f45a35e8bb5a03d699ac65007272c32ab0eded1631a8b605a43ff5bed8086072ba1e7cc2358baeca134c825a7") == 0, path, 3, 0); + sha512_of((const unsigned char *)two_block, sizeof two_block - 1, 0, true, d); + hex(d, 48, text); + check(strcmp(text, "09330c33f71147e83d192fc782cd1b4753111b173b3b05d22fa08086e3b0f712fcc7c71a557e2db966c3e9fa91746039") == 0, path, sizeof two_block - 1, 0); + static unsigned char million[1000000]; + memset(million, 'a', sizeof million); + sha512_of(million, sizeof million, 0, false, d); + hex(d, 64, text); + check(strcmp(text, "e718483d0ce769644e2e42c7bc15b4638e1f98b13b2044285632a803afa973ebde0ff244877ea60a4cb0432ce577c31beb009c5c2c49aa2e4eadb217ad8cc09b") == 0, path, sizeof million, 0); + sha512_of(million, sizeof million, 0, true, d); + hex(d, 48, text); + check(strcmp(text, "9d0e1809716474cb086e834e310a4a1ced149e9c00f248527972cec5704c2a5b07b8b3dc38ecc4ebae97ddd87f3d8985") == 0, path, sizeof million, 0); +} + +/* Selects the SHA-512 kernel: 0 the portable blocks, otherwise the CPU's + * kernel. Returns false when the CPU has none. */ +static bool select512(int kernel) { +#ifdef SCR_SHA_NI + scr_sha512_x86_state = 0; + int best = scr_sha512_x86(); + if (kernel > best - 1) return false; + scr_sha512_x86_state = kernel == 0 ? 1 : kernel + 1; + return true; +#elif defined(SCR_SHA512_ARM) + scr_sha512_arm_state = 0; + bool available = scr_sha512_arm(); + if (kernel > (available ? 1 : 0)) return false; + scr_sha512_arm_state = kernel == 0 ? 2 : 1; + return true; +#else + return kernel == 0; +#endif +} + +static const char *const kernel512_names[] = {"portable", "avx2-or-arm", "avx512"}; + static void vectors(const char *path) { unsigned char d[32]; char text[65]; @@ -108,8 +175,31 @@ int main(void) { vectors("ni"); } #endif + /* SHA-512 and SHA-384: every kernel against the portable blocks, and + * the known vectors through each. */ + int kernels512 = 0; + for (int k = 0; k < 3 && select512(k); k++) { + kernels512 = k + 1; + vectors512(kernel512_names[k]); + } + for (int k = 1; k < kernels512; k++) { + for (size_t len = 0; len <= 1100 + 1; len++) { + size_t n = len == 1101 ? BIG : len; + for (size_t c = 0; c < sizeof chunks / sizeof chunks[0]; c++) { + for (int sha384 = 0; sha384 < 2; sha384++) { + unsigned char a[64], b[64]; + select512(k); + sha512_of(data, n, chunks[c], sha384, a); + select512(0); + sha512_of(data, n, chunks[c], sha384, b); + check(memcmp(a, b, 64) == 0, kernel512_names[k], n, chunks[c]); + } + } + } + } free(data); fprintf(stderr, "sha-ni %s\n", ni ? "available" : "absent"); + fprintf(stderr, "sha-512 kernels %d\n", kernels512); fprintf(stderr, "%ld/%ld cases passed\n", total - failed, total); return failed != 0; } diff --git a/tests/corpus/5779-sha512-lengths.ts b/tests/corpus/5779-sha512-lengths.ts new file mode 100644 index 000000000..277408409 --- /dev/null +++ b/tests/corpus/5779-sha512-lengths.ts @@ -0,0 +1,35 @@ +// SHA-384 and SHA-512 over every length from 0 to 520 bytes, fed whole and +// in 1-, 7-, 100- and 128-byte updates, then over 1 MiB, and HMAC-SHA512 +// over a few lengths. Each length's digests are folded into one running +// SHA-512 so the output stays short. The CPU's SHA-512 kernel (vector or +// SHA-512 instructions) and the portable blocks give the same digests. +import { createHash, createHmac } from "node:crypto"; + +const data = Buffer.alloc(1 << 20); +let x = 0x9e3779b9; +for (let i = 0; i < data.length; i++) { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + data[i] = x & 0xff; +} + +function digest(algorithm: string, input: Buffer, chunk: number): Buffer { + const hash = createHash(algorithm); + if (chunk === 0) hash.update(input); + else for (let at = 0; at < input.length; at += chunk) hash.update(input.subarray(at, at + chunk)); + return hash.digest(); +} + +for (const algorithm of ["sha384", "sha512"]) { + for (const chunk of [0, 1, 7, 100, 128]) { + const fold = createHash("sha512"); + for (let length = 0; length <= 520; length++) fold.update(digest(algorithm, data.subarray(0, length), chunk)); + console.log(algorithm, "chunk", chunk, fold.digest("hex").slice(0, 32)); + } + console.log(algorithm, "1 MiB", digest(algorithm, data, 0).toString("hex").slice(0, 32), digest(algorithm, data, 4096).toString("hex").slice(0, 32)); +} +console.log("abc", createHash("sha512").update("abc").digest("hex")); +for (const length of [0, 1, 111, 112, 127, 128, 129, 255, 256, 1000]) { + console.log("hmac", length, createHmac("sha512", "key").update(data.subarray(0, length)).digest("hex").slice(0, 32)); +} diff --git a/tests/harness/corpus-costs/5779-sha512-lengths.ts.json b/tests/harness/corpus-costs/5779-sha512-lengths.ts.json new file mode 100644 index 000000000..9e04a316c --- /dev/null +++ b/tests/harness/corpus-costs/5779-sha512-lengths.ts.json @@ -0,0 +1 @@ +{"plain":2610,"san":3540}