mirror of
https://github.com/vercel-labs/scriptc.git
synced 2026-10-26 06:34:20 +08:00
Run SHA-384 and SHA-512 blocks with AVX2, AVX-512VL or the arm64 SHA-512 instructions
SHA-384, SHA-512, SHA-512/224 and SHA-512/256 (createHash, createHmac, crypto.hash, pbkdf2, hkdf) process their blocks with the fastest kernel the CPU supports: - x86-64 with AVX2 and BMI2: the message schedule is computed two words at a time in vector registers, interleaved with the scalar rounds, which rotate with RORX. - x86-64 with AVX-512VL: the same, with vector rotates and three-input XOR in the schedule. - arm64 with the SHA-512 extension: SHA512H, SHA512H2, SHA512SU0 and SHA512SU1. - Otherwise the portable blocks, now unrolled over a 16-word schedule ring. The CPU and OS checks run once and are cached. The update and final paths pass all their whole blocks to the kernel in one call. test_sha_kernels.c checks the FIPS 180-4 SHA-384 and SHA-512 vectors through every kernel the CPU supports and compares each kernel with the portable blocks over every length from 0 to 1100 bytes and 1 MiB, whole and in 1-, 7-, 64- and 100-byte updates. Corpus 5779 compares SHA-384, SHA-512 and HMAC-SHA512 digests over many lengths and update sizes with Node; tests/harness/corpus-costs/ records its plain and sanitized costs.
This commit is contained in:
Chris Tate
+6
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"order": [
|
||||
"<repo>/tests/corpus/5779-sha512-lengths.ts"
|
||||
],
|
||||
"diags": []
|
||||
}
|
||||
+317
-27
@@ -7405,29 +7405,319 @@ static uint64_t scr_sha512_rotr(uint64_t x, unsigned n) {
|
||||
return (x >> n) | (x << (64 - n));
|
||||
}
|
||||
|
||||
SCR_DIGEST_ALIGNED static void scr_sha512_block(uint64_t h[8], const unsigned char *data) {
|
||||
uint64_t w[80];
|
||||
for (size_t i = 0; i < 16; i++) {
|
||||
w[i] = 0;
|
||||
for (size_t b = 0; b < 8; b++) w[i] = (w[i] << 8) | data[i * 8 + b];
|
||||
#define SCR_SHA512_BSIG0(x) \
|
||||
(scr_sha512_rotr(x, 28) ^ scr_sha512_rotr(x, 34) ^ scr_sha512_rotr(x, 39))
|
||||
#define SCR_SHA512_BSIG1(x) \
|
||||
(scr_sha512_rotr(x, 14) ^ scr_sha512_rotr(x, 18) ^ scr_sha512_rotr(x, 41))
|
||||
#define SCR_SHA512_SSIG0(x) (scr_sha512_rotr(x, 1) ^ scr_sha512_rotr(x, 8) ^ ((x) >> 7))
|
||||
#define SCR_SHA512_SSIG1(x) (scr_sha512_rotr(x, 19) ^ scr_sha512_rotr(x, 61) ^ ((x) >> 6))
|
||||
|
||||
/* One round with the eight working variables passed in rotated order, so
|
||||
* an unrolled sequence of rounds needs no register moves. */
|
||||
#define SCR_SHA512_ROUND(a, b, c, d, e, f, g, h, k, w) \
|
||||
do { \
|
||||
uint64_t t1 = (h) + SCR_SHA512_BSIG1(e) + ((g) ^ ((e) & ((f) ^ (g)))) + (k) + (w); \
|
||||
(d) += t1; \
|
||||
(h) = t1 + SCR_SHA512_BSIG0(a) + (((a) & (b)) | ((c) & ((a) | (b)))); \
|
||||
} while (0)
|
||||
|
||||
static uint64_t scr_sha512_load(const unsigned char *p) {
|
||||
uint64_t v = 0;
|
||||
for (size_t b = 0; b < 8; b++) v = (v << 8) | p[b];
|
||||
return v;
|
||||
}
|
||||
|
||||
/* The message schedule is kept as a 16-word ring and extended 16 words
|
||||
* at a time between groups of 16 rounds. */
|
||||
SCR_DIGEST_ALIGNED static void scr_sha512_blocks_portable(uint64_t state[8], const unsigned char *p,
|
||||
size_t blocks) {
|
||||
uint64_t a = state[0], b = state[1], c = state[2], d = state[3];
|
||||
uint64_t e = state[4], f = state[5], g = state[6], h = state[7];
|
||||
for (; blocks != 0; blocks--, p += 128) {
|
||||
uint64_t w[16];
|
||||
for (size_t i = 0; i < 16; i++) w[i] = scr_sha512_load(p + i * 8);
|
||||
for (size_t i = 0; i < 80; i += 16) {
|
||||
if (i != 0) {
|
||||
for (size_t j = 0; j < 16; j++) {
|
||||
w[j] += SCR_SHA512_SSIG1(w[(j + 14) & 15]) + w[(j + 9) & 15] +
|
||||
SCR_SHA512_SSIG0(w[(j + 1) & 15]);
|
||||
}
|
||||
}
|
||||
const uint64_t *k = scr_sha512_k + i;
|
||||
SCR_SHA512_ROUND(a, b, c, d, e, f, g, h, k[0], w[0]);
|
||||
SCR_SHA512_ROUND(h, a, b, c, d, e, f, g, k[1], w[1]);
|
||||
SCR_SHA512_ROUND(g, h, a, b, c, d, e, f, k[2], w[2]);
|
||||
SCR_SHA512_ROUND(f, g, h, a, b, c, d, e, k[3], w[3]);
|
||||
SCR_SHA512_ROUND(e, f, g, h, a, b, c, d, k[4], w[4]);
|
||||
SCR_SHA512_ROUND(d, e, f, g, h, a, b, c, k[5], w[5]);
|
||||
SCR_SHA512_ROUND(c, d, e, f, g, h, a, b, k[6], w[6]);
|
||||
SCR_SHA512_ROUND(b, c, d, e, f, g, h, a, k[7], w[7]);
|
||||
SCR_SHA512_ROUND(a, b, c, d, e, f, g, h, k[8], w[8]);
|
||||
SCR_SHA512_ROUND(h, a, b, c, d, e, f, g, k[9], w[9]);
|
||||
SCR_SHA512_ROUND(g, h, a, b, c, d, e, f, k[10], w[10]);
|
||||
SCR_SHA512_ROUND(f, g, h, a, b, c, d, e, k[11], w[11]);
|
||||
SCR_SHA512_ROUND(e, f, g, h, a, b, c, d, k[12], w[12]);
|
||||
SCR_SHA512_ROUND(d, e, f, g, h, a, b, c, k[13], w[13]);
|
||||
SCR_SHA512_ROUND(c, d, e, f, g, h, a, b, k[14], w[14]);
|
||||
SCR_SHA512_ROUND(b, c, d, e, f, g, h, a, k[15], w[15]);
|
||||
}
|
||||
a = state[0] += a; b = state[1] += b; c = state[2] += c; d = state[3] += d;
|
||||
e = state[4] += e; f = state[5] += f; g = state[6] += g; h = state[7] += h;
|
||||
}
|
||||
for (size_t i = 16; i < 80; i++) {
|
||||
uint64_t s0 = scr_sha512_rotr(w[i - 15], 1) ^ scr_sha512_rotr(w[i - 15], 8) ^ (w[i - 15] >> 7);
|
||||
uint64_t s1 = scr_sha512_rotr(w[i - 2], 19) ^ scr_sha512_rotr(w[i - 2], 61) ^ (w[i - 2] >> 6);
|
||||
w[i] = w[i - 16] + s0 + w[i - 7] + s1;
|
||||
}
|
||||
|
||||
#ifdef SCR_SHA_NI
|
||||
/* ── SHA-512 kernels (x86-64) ─────────────────────────────────────────
|
||||
* With AVX2 and BMI2, the message schedule is computed two words at a
|
||||
* time in vector registers, interleaved with the scalar rounds, and the
|
||||
* rounds rotate with RORX. With AVX-512VL the schedule uses vector
|
||||
* rotates and three-input XOR. The CPU and OS state checks run once and
|
||||
* are cached. */
|
||||
static int scr_sha512_x86_state; /* 0 unknown, 1 portable, 2 AVX2, 3 AVX-512VL */
|
||||
|
||||
static int scr_sha512_x86(void) {
|
||||
if (scr_sha512_x86_state == 0) {
|
||||
unsigned a, b, c, d;
|
||||
int kind = 1;
|
||||
if (__get_cpuid(1, &a, &b, &c, &d) && (c & bit_OSXSAVE) && (c & bit_AVX)) {
|
||||
unsigned lo, hi;
|
||||
__asm__ volatile("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0));
|
||||
(void)hi;
|
||||
/* XMM and YMM state enabled. */
|
||||
if ((lo & 6) == 6 && __get_cpuid_count(7, 0, &a, &b, &c, &d) && (b & bit_AVX2) &&
|
||||
(b & bit_BMI2)) {
|
||||
kind = 2;
|
||||
/* Opmask, ZMM0-15 upper halves and ZMM16-31 state enabled. */
|
||||
if ((b & bit_AVX512F) && (b & bit_AVX512VL) && (lo & 0xe0) == 0xe0) kind = 3;
|
||||
}
|
||||
}
|
||||
scr_sha512_x86_state = kind;
|
||||
}
|
||||
uint64_t a = h[0], b = h[1], c = h[2], d = h[3];
|
||||
uint64_t e = h[4], f = h[5], g = h[6], v = h[7];
|
||||
for (size_t i = 0; i < 80; i++) {
|
||||
uint64_t s1 = scr_sha512_rotr(e, 14) ^ scr_sha512_rotr(e, 18) ^ scr_sha512_rotr(e, 41);
|
||||
uint64_t t1 = v + s1 + ((e & f) ^ (~e & g)) + scr_sha512_k[i] + w[i];
|
||||
uint64_t s0 = scr_sha512_rotr(a, 28) ^ scr_sha512_rotr(a, 34) ^ scr_sha512_rotr(a, 39);
|
||||
uint64_t t2 = s0 + ((a & b) ^ (a & c) ^ (b & c));
|
||||
v = g; g = f; f = e; e = d + t1;
|
||||
d = c; c = b; b = a; a = t1 + t2;
|
||||
return scr_sha512_x86_state;
|
||||
}
|
||||
|
||||
__attribute__((target("avx2,bmi2"))) static inline __m128i scr_sha512_ssig0_avx2(__m128i x) {
|
||||
return _mm_xor_si128(_mm_xor_si128(_mm_or_si128(_mm_srli_epi64(x, 1), _mm_slli_epi64(x, 63)),
|
||||
_mm_or_si128(_mm_srli_epi64(x, 8), _mm_slli_epi64(x, 56))),
|
||||
_mm_srli_epi64(x, 7));
|
||||
}
|
||||
|
||||
__attribute__((target("avx2,bmi2"))) static inline __m128i scr_sha512_ssig1_avx2(__m128i x) {
|
||||
return _mm_xor_si128(_mm_xor_si128(_mm_or_si128(_mm_srli_epi64(x, 19), _mm_slli_epi64(x, 45)),
|
||||
_mm_or_si128(_mm_srli_epi64(x, 61), _mm_slli_epi64(x, 3))),
|
||||
_mm_srli_epi64(x, 6));
|
||||
}
|
||||
|
||||
__attribute__((target("avx512f,avx512vl,bmi2"))) static inline __m128i
|
||||
scr_sha512_ssig0_avx512(__m128i x) {
|
||||
return _mm_ternarylogic_epi64(_mm_ror_epi64(x, 1), _mm_ror_epi64(x, 8), _mm_srli_epi64(x, 7),
|
||||
0x96);
|
||||
}
|
||||
|
||||
__attribute__((target("avx512f,avx512vl,bmi2"))) static inline __m128i
|
||||
scr_sha512_ssig1_avx512(__m128i x) {
|
||||
return _mm_ternarylogic_epi64(_mm_ror_epi64(x, 19), _mm_ror_epi64(x, 61), _mm_srli_epi64(x, 6),
|
||||
0x96);
|
||||
}
|
||||
|
||||
/* A round for the vector kernels. wk is the schedule word plus its
|
||||
* constant. Maj(a, b, c) is ((a ^ b) & (b ^ c)) ^ b, where b ^ c is the
|
||||
* previous round's a ^ b. The empty asm statements keep the compiler from
|
||||
* reassociating the additions: adding Sigma1(e) last keeps the sum of h,
|
||||
* wk and Ch off the dependency chain through e. */
|
||||
#define SCR_SHA512_PIN(v) __asm__("" : "+r"(v))
|
||||
#define SCR_SHA512_ROUND_WK(a, b, c, d, e, f, g, h, wk, ab, bc) \
|
||||
do { \
|
||||
uint64_t t1 = (h) + (wk); \
|
||||
SCR_SHA512_PIN(t1); \
|
||||
t1 += (g) ^ ((e) & ((f) ^ (g))); \
|
||||
SCR_SHA512_PIN(t1); \
|
||||
t1 += SCR_SHA512_BSIG1(e); \
|
||||
SCR_SHA512_PIN(t1); \
|
||||
(d) += t1; \
|
||||
(ab) = (a) ^ (b); \
|
||||
uint64_t m = t1 + (((ab) & (bc)) ^ (b)); \
|
||||
SCR_SHA512_PIN(m); \
|
||||
(h) = m + SCR_SHA512_BSIG0(a); \
|
||||
} while (0)
|
||||
|
||||
/* Words 2j and 2j+1 of the schedule, in the ring slot of words 2j-16 and
|
||||
* 2j-15, and their sum with the round constants into wk. */
|
||||
#define SCR_SHA512_SCHEDULE(j, ssig0, ssig1) \
|
||||
do { \
|
||||
if ((j) < 40) { \
|
||||
__m128i s15 = _mm_alignr_epi8(w[((j) + 1) & 7], w[(j) & 7], 8); \
|
||||
__m128i s7 = _mm_alignr_epi8(w[((j) + 5) & 7], w[((j) + 4) & 7], 8); \
|
||||
__m128i next = _mm_add_epi64(_mm_add_epi64(w[(j) & 7], ssig0(s15)), \
|
||||
_mm_add_epi64(s7, ssig1(w[((j) + 7) & 7]))); \
|
||||
w[(j) & 7] = next; \
|
||||
_mm_store_si128((__m128i *)&wk[(j) * 2], \
|
||||
_mm_add_epi64(next, _mm_loadu_si128((const __m128i *)&scr_sha512_k[(j) * 2]))); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/* Rounds 2i to 2i+7 compute the schedule words 2i+16 to 2i+23. */
|
||||
#define SCR_SHA512_VECTOR_BODY(ssig0, ssig1) \
|
||||
const __m128i bswap = _mm_set_epi64x(0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL); \
|
||||
uint64_t a = state[0], b = state[1], c = state[2], d = state[3]; \
|
||||
uint64_t e = state[4], f = state[5], g = state[6], h = state[7]; \
|
||||
for (; blocks != 0; blocks--, p += 128) { \
|
||||
__m128i w[8]; \
|
||||
uint64_t wk[80] __attribute__((aligned(16))); \
|
||||
_Pragma("GCC unroll 8") for (int i = 0; i < 8; i++) { \
|
||||
w[i] = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(p + i * 16)), bswap); \
|
||||
_mm_store_si128((__m128i *)&wk[i * 2], \
|
||||
_mm_add_epi64(w[i], _mm_loadu_si128((const __m128i *)&scr_sha512_k[i * 2]))); \
|
||||
} \
|
||||
uint64_t x = b ^ c, y; \
|
||||
_Pragma("GCC unroll 10") for (int i = 0; i < 40; i += 4) { \
|
||||
SCR_SHA512_SCHEDULE(i + 8, ssig0, ssig1); \
|
||||
SCR_SHA512_ROUND_WK(a, b, c, d, e, f, g, h, wk[2 * i + 0], y, x); \
|
||||
SCR_SHA512_ROUND_WK(h, a, b, c, d, e, f, g, wk[2 * i + 1], x, y); \
|
||||
SCR_SHA512_SCHEDULE(i + 9, ssig0, ssig1); \
|
||||
SCR_SHA512_ROUND_WK(g, h, a, b, c, d, e, f, wk[2 * i + 2], y, x); \
|
||||
SCR_SHA512_ROUND_WK(f, g, h, a, b, c, d, e, wk[2 * i + 3], x, y); \
|
||||
SCR_SHA512_SCHEDULE(i + 10, ssig0, ssig1); \
|
||||
SCR_SHA512_ROUND_WK(e, f, g, h, a, b, c, d, wk[2 * i + 4], y, x); \
|
||||
SCR_SHA512_ROUND_WK(d, e, f, g, h, a, b, c, wk[2 * i + 5], x, y); \
|
||||
SCR_SHA512_SCHEDULE(i + 11, ssig0, ssig1); \
|
||||
SCR_SHA512_ROUND_WK(c, d, e, f, g, h, a, b, wk[2 * i + 6], y, x); \
|
||||
SCR_SHA512_ROUND_WK(b, c, d, e, f, g, h, a, wk[2 * i + 7], x, y); \
|
||||
} \
|
||||
a = state[0] += a; b = state[1] += b; c = state[2] += c; d = state[3] += d; \
|
||||
e = state[4] += e; f = state[5] += f; g = state[6] += g; h = state[7] += h; \
|
||||
}
|
||||
h[0] += a; h[1] += b; h[2] += c; h[3] += d;
|
||||
h[4] += e; h[5] += f; h[6] += g; h[7] += v;
|
||||
|
||||
__attribute__((target("avx2,bmi2"), aligned(64))) static void
|
||||
scr_sha512_blocks_avx2(uint64_t state[8], const unsigned char *p, size_t blocks) {
|
||||
SCR_SHA512_VECTOR_BODY(scr_sha512_ssig0_avx2, scr_sha512_ssig1_avx2)
|
||||
}
|
||||
|
||||
__attribute__((target("avx512f,avx512vl,bmi2"), aligned(64))) static void
|
||||
scr_sha512_blocks_avx512(uint64_t state[8], const unsigned char *p, size_t blocks) {
|
||||
SCR_SHA512_VECTOR_BODY(scr_sha512_ssig0_avx512, scr_sha512_ssig1_avx512)
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
#if defined(__aarch64__) && (defined(__clang__) || defined(__GNUC__)) && !defined(_WIN32)
|
||||
/* ── SHA-512 kernel (arm64) ───────────────────────────────────────────
|
||||
* On CPUs with the SHA-512 extension (FEAT_SHA512), the blocks run with
|
||||
* SHA512H, SHA512H2, SHA512SU0 and SHA512SU1; other CPUs use the portable
|
||||
* blocks. The CPU check runs once and is cached. */
|
||||
#include <arm_neon.h>
|
||||
#if defined(__APPLE__)
|
||||
#include <sys/sysctl.h>
|
||||
#else
|
||||
#include <sys/auxv.h>
|
||||
#endif
|
||||
#define SCR_SHA512_ARM 1
|
||||
#if defined(__clang__)
|
||||
#define SCR_SHA512_ARM_TARGET __attribute__((target("sha3")))
|
||||
#else
|
||||
#define SCR_SHA512_ARM_TARGET __attribute__((target("+sha3")))
|
||||
#endif
|
||||
|
||||
static int scr_sha512_arm_state; /* 0 unknown, 1 available, 2 absent */
|
||||
|
||||
static bool scr_sha512_arm(void) {
|
||||
if (scr_sha512_arm_state == 0) {
|
||||
bool ok;
|
||||
#if defined(__APPLE__)
|
||||
int value = 0;
|
||||
size_t size = sizeof value;
|
||||
ok = sysctlbyname("hw.optional.armv8_2_sha512", &value, &size, NULL, 0) == 0 && value != 0;
|
||||
#else
|
||||
ok = (getauxval(AT_HWCAP) & (1ul << 21)) != 0; /* HWCAP_SHA512 */
|
||||
#endif
|
||||
scr_sha512_arm_state = ok ? 1 : 2;
|
||||
}
|
||||
return scr_sha512_arm_state == 1;
|
||||
}
|
||||
|
||||
/* Two rounds per step. With the state pairs in the roles (A, B, C, D) =
|
||||
* (ab, cd, ef, gh), a step computes I = SHA512H(swap(K + W) + D, ext(C, D),
|
||||
* ext(B, C)), D = SHA512H2(I, B, A) and B += I; the roles then rotate to
|
||||
* (D, A, B, C). */
|
||||
#define SCR_SHA512_ARM_STEP(A, B, C, D, msg, k) \
|
||||
do { \
|
||||
uint64x2_t kw = vaddq_u64((msg), vld1q_u64(k)); \
|
||||
uint64x2_t sum = vaddq_u64(vextq_u64(kw, kw, 1), (D)); \
|
||||
uint64x2_t i = vsha512hq_u64(sum, vextq_u64((C), (D), 1), vextq_u64((B), (C), 1)); \
|
||||
(D) = vsha512h2q_u64(i, (B), (A)); \
|
||||
(B) = vaddq_u64((B), i); \
|
||||
} while (0)
|
||||
|
||||
/* Message words 2j and 2j+1 from the eight pairs before them. */
|
||||
#define SCR_SHA512_ARM_SCHEDULE(m0, m1, m4, m5, m7) \
|
||||
(m0) = vsha512su1q_u64(vsha512su0q_u64((m0), (m1)), (m7), vextq_u64((m4), (m5), 1))
|
||||
|
||||
SCR_SHA512_ARM_TARGET static void scr_sha512_blocks_arm(uint64_t state[8], const unsigned char *p,
|
||||
size_t blocks) {
|
||||
uint64x2_t ab = vld1q_u64(&state[0]), cd = vld1q_u64(&state[2]);
|
||||
uint64x2_t ef = vld1q_u64(&state[4]), gh = vld1q_u64(&state[6]);
|
||||
for (; blocks != 0; blocks--, p += 128) {
|
||||
uint64x2_t ab0 = ab, cd0 = cd, ef0 = ef, gh0 = gh;
|
||||
uint64x2_t m0 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 0)));
|
||||
uint64x2_t m1 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 16)));
|
||||
uint64x2_t m2 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 32)));
|
||||
uint64x2_t m3 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 48)));
|
||||
uint64x2_t m4 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 64)));
|
||||
uint64x2_t m5 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 80)));
|
||||
uint64x2_t m6 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 96)));
|
||||
uint64x2_t m7 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p + 112)));
|
||||
const uint64_t *k = scr_sha512_k;
|
||||
for (int i = 0; i < 5; i++, k += 16) {
|
||||
SCR_SHA512_ARM_STEP(ab, cd, ef, gh, m0, k + 0);
|
||||
SCR_SHA512_ARM_STEP(gh, ab, cd, ef, m1, k + 2);
|
||||
SCR_SHA512_ARM_STEP(ef, gh, ab, cd, m2, k + 4);
|
||||
SCR_SHA512_ARM_STEP(cd, ef, gh, ab, m3, k + 6);
|
||||
SCR_SHA512_ARM_STEP(ab, cd, ef, gh, m4, k + 8);
|
||||
SCR_SHA512_ARM_STEP(gh, ab, cd, ef, m5, k + 10);
|
||||
SCR_SHA512_ARM_STEP(ef, gh, ab, cd, m6, k + 12);
|
||||
SCR_SHA512_ARM_STEP(cd, ef, gh, ab, m7, k + 14);
|
||||
if (i == 4) break;
|
||||
SCR_SHA512_ARM_SCHEDULE(m0, m1, m4, m5, m7);
|
||||
SCR_SHA512_ARM_SCHEDULE(m1, m2, m5, m6, m0);
|
||||
SCR_SHA512_ARM_SCHEDULE(m2, m3, m6, m7, m1);
|
||||
SCR_SHA512_ARM_SCHEDULE(m3, m4, m7, m0, m2);
|
||||
SCR_SHA512_ARM_SCHEDULE(m4, m5, m0, m1, m3);
|
||||
SCR_SHA512_ARM_SCHEDULE(m5, m6, m1, m2, m4);
|
||||
SCR_SHA512_ARM_SCHEDULE(m6, m7, m2, m3, m5);
|
||||
SCR_SHA512_ARM_SCHEDULE(m7, m0, m3, m4, m6);
|
||||
}
|
||||
ab = vaddq_u64(ab, ab0);
|
||||
cd = vaddq_u64(cd, cd0);
|
||||
ef = vaddq_u64(ef, ef0);
|
||||
gh = vaddq_u64(gh, gh0);
|
||||
}
|
||||
vst1q_u64(&state[0], ab);
|
||||
vst1q_u64(&state[2], cd);
|
||||
vst1q_u64(&state[4], ef);
|
||||
vst1q_u64(&state[6], gh);
|
||||
}
|
||||
#endif
|
||||
|
||||
static void scr_sha512_blocks(uint64_t h[8], const unsigned char *p, size_t blocks) {
|
||||
#ifdef SCR_SHA_NI
|
||||
switch (scr_sha512_x86()) {
|
||||
case 3:
|
||||
scr_sha512_blocks_avx512(h, p, blocks);
|
||||
return;
|
||||
case 2:
|
||||
scr_sha512_blocks_avx2(h, p, blocks);
|
||||
return;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
#ifdef SCR_SHA512_ARM
|
||||
if (scr_sha512_arm()) {
|
||||
scr_sha512_blocks_arm(h, p, blocks);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
scr_sha512_blocks_portable(h, p, blocks);
|
||||
}
|
||||
|
||||
static void scr_sha512_init(ScrSha512Ctx *ctx, bool sha384) {
|
||||
@@ -7456,14 +7746,15 @@ static void scr_sha512_update(ScrSha512Ctx *ctx, const unsigned char *data, size
|
||||
data += take;
|
||||
len -= take;
|
||||
if (ctx->tail_len == 128) {
|
||||
scr_sha512_block(ctx->h, ctx->tail);
|
||||
scr_sha512_blocks(ctx->h, ctx->tail, 1);
|
||||
ctx->tail_len = 0;
|
||||
}
|
||||
}
|
||||
while (len >= 128) {
|
||||
scr_sha512_block(ctx->h, data);
|
||||
data += 128;
|
||||
len -= 128;
|
||||
if (len >= 128) {
|
||||
size_t blocks = len / 128;
|
||||
scr_sha512_blocks(ctx->h, data, blocks);
|
||||
data += blocks * 128;
|
||||
len -= blocks * 128;
|
||||
}
|
||||
if (len != 0) {
|
||||
memcpy(ctx->tail, data, len);
|
||||
@@ -7482,8 +7773,7 @@ static size_t scr_sha512_final(const ScrSha512Ctx *source, unsigned char out[64]
|
||||
tail[pad - 1 - b] = (unsigned char)(low >> (8 * b));
|
||||
tail[pad - 9 - b] = (unsigned char)(high >> (8 * b));
|
||||
}
|
||||
scr_sha512_block(ctx.h, tail);
|
||||
if (pad == 256) scr_sha512_block(ctx.h, tail + 128);
|
||||
scr_sha512_blocks(ctx.h, tail, pad / 128);
|
||||
size_t len = sha384 ? 48 : 64;
|
||||
for (size_t i = 0; i < len; i++) out[i] = (unsigned char)(ctx.h[i / 8] >> (56 - 8 * (i % 8)));
|
||||
return len;
|
||||
|
||||
@@ -42,7 +42,7 @@ const runtimeSources = [
|
||||
];
|
||||
|
||||
// Compiled once with ASan, UBSan and the RC audit. The test runs the SHA-NI and portable
|
||||
// SHA-1/SHA-256 blocks over the same inputs and checks known vectors.
|
||||
// SHA-1/SHA-256 blocks and the SHA-512 kernels over the same inputs and checks known vectors.
|
||||
beforeAll(async () => {
|
||||
await mkdir(join(testDir, "build"), { recursive: true });
|
||||
await execFileAsync("clang", [
|
||||
@@ -64,7 +64,7 @@ beforeAll(async () => {
|
||||
]);
|
||||
}, 300_000);
|
||||
|
||||
test("SHA-1 and SHA-256: the SHA-NI blocks match the portable blocks", async () => {
|
||||
test("SHA-1, SHA-256, SHA-384 and SHA-512: the CPU kernels match the portable blocks", async () => {
|
||||
const { stderr } = await execFileAsync(bin, [], {
|
||||
env: { ...process.env, UBSAN_OPTIONS: "halt_on_error=1" },
|
||||
});
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
/* SHA-1 and SHA-256 block kernels (scr_lib.c), built by
|
||||
/* SHA-1, SHA-256, SHA-384 and SHA-512 block kernels (scr_lib.c), built by
|
||||
* sha-kernels.test.ts. The unit is included directly so the test can run
|
||||
* the SHA-NI blocks and the portable blocks over the same inputs: known
|
||||
* vectors, every length from 0 to 1100 bytes, and 1 MiB, each fed whole
|
||||
* and in 1-, 7-, 64- and 100-byte updates. On CPUs without the SHA
|
||||
* extensions only the portable blocks run. */
|
||||
* each kernel the CPU supports and the portable blocks over the same
|
||||
* inputs: known vectors, every length from 0 to 1100 bytes, and 1 MiB,
|
||||
* each fed whole and in 1-, 7-, 64- and 100-byte updates. The SHA-1 and
|
||||
* SHA-256 kernels use the SHA extensions; the SHA-512 kernels use AVX2 or
|
||||
* AVX-512VL on x86-64 and the SHA-512 extension on arm64. On CPUs without
|
||||
* them only the portable blocks run. */
|
||||
#include "../src/scr_lib.c"
|
||||
|
||||
#include <stdio.h>
|
||||
@@ -45,6 +47,71 @@ static void sha1_of(const unsigned char *data, size_t len, size_t chunk, unsigne
|
||||
scr_sha1_final(&ctx, out);
|
||||
}
|
||||
|
||||
static void sha512_of(const unsigned char *data, size_t len, size_t chunk, bool sha384,
|
||||
unsigned char out[64]) {
|
||||
ScrSha512Ctx ctx;
|
||||
scr_sha512_init(&ctx, sha384);
|
||||
for (size_t at = 0; at < len;) {
|
||||
size_t take = chunk == 0 || chunk > len - at ? len - at : chunk;
|
||||
scr_sha512_update(&ctx, data + at, take);
|
||||
at += take;
|
||||
}
|
||||
scr_sha512_final(&ctx, out, sha384);
|
||||
}
|
||||
|
||||
/* FIPS 180-4 example vectors for SHA-384 and SHA-512. */
|
||||
static void vectors512(const char *path) {
|
||||
unsigned char d[64];
|
||||
char text[129];
|
||||
static const char two_block[] =
|
||||
"abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu";
|
||||
sha512_of((const unsigned char *)"abc", 3, 0, false, d);
|
||||
hex(d, 64, text);
|
||||
check(strcmp(text, "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f") == 0, path, 3, 0);
|
||||
sha512_of((const unsigned char *)"", 0, 0, false, d);
|
||||
hex(d, 64, text);
|
||||
check(strcmp(text, "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e") == 0, path, 0, 0);
|
||||
sha512_of((const unsigned char *)two_block, sizeof two_block - 1, 0, false, d);
|
||||
hex(d, 64, text);
|
||||
check(strcmp(text, "8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa17299aeadb6889018501d289e4900f7e4331b99dec4b5433ac7d329eeb6dd26545e96e55b874be909") == 0, path, sizeof two_block - 1, 0);
|
||||
sha512_of((const unsigned char *)"abc", 3, 0, true, d);
|
||||
hex(d, 48, text);
|
||||
check(strcmp(text, "cb00753f45a35e8bb5a03d699ac65007272c32ab0eded1631a8b605a43ff5bed8086072ba1e7cc2358baeca134c825a7") == 0, path, 3, 0);
|
||||
sha512_of((const unsigned char *)two_block, sizeof two_block - 1, 0, true, d);
|
||||
hex(d, 48, text);
|
||||
check(strcmp(text, "09330c33f71147e83d192fc782cd1b4753111b173b3b05d22fa08086e3b0f712fcc7c71a557e2db966c3e9fa91746039") == 0, path, sizeof two_block - 1, 0);
|
||||
static unsigned char million[1000000];
|
||||
memset(million, 'a', sizeof million);
|
||||
sha512_of(million, sizeof million, 0, false, d);
|
||||
hex(d, 64, text);
|
||||
check(strcmp(text, "e718483d0ce769644e2e42c7bc15b4638e1f98b13b2044285632a803afa973ebde0ff244877ea60a4cb0432ce577c31beb009c5c2c49aa2e4eadb217ad8cc09b") == 0, path, sizeof million, 0);
|
||||
sha512_of(million, sizeof million, 0, true, d);
|
||||
hex(d, 48, text);
|
||||
check(strcmp(text, "9d0e1809716474cb086e834e310a4a1ced149e9c00f248527972cec5704c2a5b07b8b3dc38ecc4ebae97ddd87f3d8985") == 0, path, sizeof million, 0);
|
||||
}
|
||||
|
||||
/* Selects the SHA-512 kernel: 0 the portable blocks, otherwise the CPU's
|
||||
* kernel. Returns false when the CPU has none. */
|
||||
static bool select512(int kernel) {
|
||||
#ifdef SCR_SHA_NI
|
||||
scr_sha512_x86_state = 0;
|
||||
int best = scr_sha512_x86();
|
||||
if (kernel > best - 1) return false;
|
||||
scr_sha512_x86_state = kernel == 0 ? 1 : kernel + 1;
|
||||
return true;
|
||||
#elif defined(SCR_SHA512_ARM)
|
||||
scr_sha512_arm_state = 0;
|
||||
bool available = scr_sha512_arm();
|
||||
if (kernel > (available ? 1 : 0)) return false;
|
||||
scr_sha512_arm_state = kernel == 0 ? 2 : 1;
|
||||
return true;
|
||||
#else
|
||||
return kernel == 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static const char *const kernel512_names[] = {"portable", "avx2-or-arm", "avx512"};
|
||||
|
||||
static void vectors(const char *path) {
|
||||
unsigned char d[32];
|
||||
char text[65];
|
||||
@@ -108,8 +175,31 @@ int main(void) {
|
||||
vectors("ni");
|
||||
}
|
||||
#endif
|
||||
/* SHA-512 and SHA-384: every kernel against the portable blocks, and
|
||||
* the known vectors through each. */
|
||||
int kernels512 = 0;
|
||||
for (int k = 0; k < 3 && select512(k); k++) {
|
||||
kernels512 = k + 1;
|
||||
vectors512(kernel512_names[k]);
|
||||
}
|
||||
for (int k = 1; k < kernels512; k++) {
|
||||
for (size_t len = 0; len <= 1100 + 1; len++) {
|
||||
size_t n = len == 1101 ? BIG : len;
|
||||
for (size_t c = 0; c < sizeof chunks / sizeof chunks[0]; c++) {
|
||||
for (int sha384 = 0; sha384 < 2; sha384++) {
|
||||
unsigned char a[64], b[64];
|
||||
select512(k);
|
||||
sha512_of(data, n, chunks[c], sha384, a);
|
||||
select512(0);
|
||||
sha512_of(data, n, chunks[c], sha384, b);
|
||||
check(memcmp(a, b, 64) == 0, kernel512_names[k], n, chunks[c]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
free(data);
|
||||
fprintf(stderr, "sha-ni %s\n", ni ? "available" : "absent");
|
||||
fprintf(stderr, "sha-512 kernels %d\n", kernels512);
|
||||
fprintf(stderr, "%ld/%ld cases passed\n", total - failed, total);
|
||||
return failed != 0;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
// SHA-384 and SHA-512 over every length from 0 to 520 bytes, fed whole and
|
||||
// in 1-, 7-, 100- and 128-byte updates, then over 1 MiB, and HMAC-SHA512
|
||||
// over a few lengths. Each length's digests are folded into one running
|
||||
// SHA-512 so the output stays short. The CPU's SHA-512 kernel (vector or
|
||||
// SHA-512 instructions) and the portable blocks give the same digests.
|
||||
import { createHash, createHmac } from "node:crypto";
|
||||
|
||||
const data = Buffer.alloc(1 << 20);
|
||||
let x = 0x9e3779b9;
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
x ^= x << 13;
|
||||
x ^= x >>> 17;
|
||||
x ^= x << 5;
|
||||
data[i] = x & 0xff;
|
||||
}
|
||||
|
||||
function digest(algorithm: string, input: Buffer, chunk: number): Buffer {
|
||||
const hash = createHash(algorithm);
|
||||
if (chunk === 0) hash.update(input);
|
||||
else for (let at = 0; at < input.length; at += chunk) hash.update(input.subarray(at, at + chunk));
|
||||
return hash.digest();
|
||||
}
|
||||
|
||||
for (const algorithm of ["sha384", "sha512"]) {
|
||||
for (const chunk of [0, 1, 7, 100, 128]) {
|
||||
const fold = createHash("sha512");
|
||||
for (let length = 0; length <= 520; length++) fold.update(digest(algorithm, data.subarray(0, length), chunk));
|
||||
console.log(algorithm, "chunk", chunk, fold.digest("hex").slice(0, 32));
|
||||
}
|
||||
console.log(algorithm, "1 MiB", digest(algorithm, data, 0).toString("hex").slice(0, 32), digest(algorithm, data, 4096).toString("hex").slice(0, 32));
|
||||
}
|
||||
console.log("abc", createHash("sha512").update("abc").digest("hex"));
|
||||
for (const length of [0, 1, 111, 112, 127, 128, 129, 255, 256, 1000]) {
|
||||
console.log("hmac", length, createHmac("sha512", "key").update(data.subarray(0, length)).digest("hex").slice(0, 32));
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"plain":2610,"san":3540}
|
||||
Reference in New Issue
Block a user