From: Chris Duncan Date: Thu, 1 Oct 2026 19:53:43 +0000 (-0700) Subject: Migrate multiply carry chain from scalar extraction and carry to direct vector manipu... X-Git-Url: https://git.codecow.com/?a=commitdiff_plain;h=26fd27aa7cf77c36fa48925566b2be6b2ef464d3;p=nano25519.git Migrate multiply carry chain from scalar extraction and carry to direct vector manipulation and storage. --- diff --git a/src/assembly/ed25519/fe.ts b/src/assembly/ed25519/fe.ts index ff29d99..3ad204b 100644 --- a/src/assembly/ed25519/fe.ts +++ b/src/assembly/ed25519/fe.ts @@ -284,6 +284,7 @@ export function fe_iszero (f: FieldElement): u8 { * @param g FieldElement multiplicand source */ export function fe_mul (h: FieldElement, f: FieldElement, g: FieldElement): void { + const h_ptr: usize = changetype(h) const f_ptr: usize = changetype(f) const g_ptr: usize = changetype(g) const g0123: v128 = v128.load(g_ptr, 0) @@ -297,10 +298,9 @@ export function fe_mul (h: FieldElement, f: FieldElement, g: FieldElement): void const odds: v128 = i32x4(0, -1, 0, -1) // factor of 19 for wrap from h9 to h0 - const v19: v128 = i32x4.splat(19) - const g0123_19: v128 = i32x4.mul(g0123, v19) - const g4567_19: v128 = i32x4.mul(g4567, v19) - const g89xx_19: v128 = i32x4.mul(g89xx, v19) + const g0123_19: v128 = i32x4.mul(g0123, i32x4.splat(19)) + const g4567_19: v128 = i32x4.mul(g4567, i32x4.splat(19)) + const g89xx_19: v128 = i32x4.mul(g89xx, i32x4.splat(19)) // f[0] let f_i: v128 = v128.load_splat(f_ptr, 0 << 2) @@ -313,7 +313,7 @@ export function fe_mul (h: FieldElement, f: FieldElement, g: FieldElement): void // f[1] f_i = v128.load_splat(f_ptr, 1 << 2) - f_i = i32x4.add(f_i, v128.and(f_i, odds)) + f_i = v128.add(f_i, v128.and(f_i, odds)) let h12: v128 = i64x2.extmul_low_i32x4_s(f_i, g0123) let h34: v128 = i64x2.extmul_high_i32x4_s(f_i, g0123) @@ -338,7 +338,7 @@ export function fe_mul (h: FieldElement, f: FieldElement, g: FieldElement): void // f[3] f_i = v128.load_splat(f_ptr, 3 << 2) - f_i = i32x4.add(f_i, v128.and(f_i, odds)) + f_i = v128.add(f_i, v128.and(f_i, odds)) t0 = i64x2.extmul_low_i32x4_s(f_i, g0123) t1 = i64x2.extmul_high_i32x4_s(f_i, g0123) @@ -369,7 +369,7 @@ export function fe_mul (h: FieldElement, f: FieldElement, g: FieldElement): void // f[5] f_i = v128.load_splat(f_ptr, 5 << 2) - f_i = i32x4.add(f_i, v128.and(f_i, odds)) + f_i = v128.add(f_i, v128.and(f_i, odds)) t0 = i64x2.extmul_low_i32x4_s(f_i, g0123) t1 = i64x2.extmul_high_i32x4_s(f_i, g0123) @@ -400,7 +400,7 @@ export function fe_mul (h: FieldElement, f: FieldElement, g: FieldElement): void // f[7] f_i = v128.load_splat(f_ptr, 7 << 2) - f_i = i32x4.add(f_i, v128.and(f_i, odds)) + f_i = v128.add(f_i, v128.and(f_i, odds)) t0 = i64x2.extmul_low_i32x4_s(f_i, g0123) t1 = i64x2.extmul_high_i32x4_s(f_i, v128.bitselect(g0123, g0123_19, m)) @@ -431,7 +431,7 @@ export function fe_mul (h: FieldElement, f: FieldElement, g: FieldElement): void // f[9] f_i = v128.load_splat(f_ptr, 9 << 2) - f_i = i32x4.add(f_i, v128.and(f_i, odds)) + f_i = v128.add(f_i, v128.and(f_i, odds)) t0 = i64x2.extmul_low_i32x4_s(f_i, v128.bitselect(g0123, g0123_19, m)) t1 = i64x2.extmul_high_i32x4_s(f_i, g0123_19) @@ -445,85 +445,57 @@ export function fe_mul (h: FieldElement, f: FieldElement, g: FieldElement): void h56 = i64x2.add(h56, t3) h78 = i64x2.add(h78, t4) - // extract scalars from vectors - let h0: i64 = i64x2.extract_lane(h90, 1) + i64x2.extract_lane(h01, 0) - let h1: i64 = i64x2.extract_lane(h01, 1) + i64x2.extract_lane(h12, 0) - let h2: i64 = i64x2.extract_lane(h12, 1) + i64x2.extract_lane(h23, 0) - let h3: i64 = i64x2.extract_lane(h23, 1) + i64x2.extract_lane(h34, 0) - let h4: i64 = i64x2.extract_lane(h34, 1) + i64x2.extract_lane(h45, 0) - let h5: i64 = i64x2.extract_lane(h45, 1) + i64x2.extract_lane(h56, 0) - let h6: i64 = i64x2.extract_lane(h56, 1) + i64x2.extract_lane(h67, 0) - let h7: i64 = i64x2.extract_lane(h67, 1) + i64x2.extract_lane(h78, 0) - let h8: i64 = i64x2.extract_lane(h78, 1) + i64x2.extract_lane(h89, 0) - let h9: i64 = i64x2.extract_lane(h89, 1) + i64x2.extract_lane(h90, 0) - - // reduce scalars with carry - let carry0: i64 - let carry1: i64 - let carry2: i64 - let carry3: i64 - let carry4: i64 - let carry5: i64 - let carry6: i64 - let carry7: i64 - let carry8: i64 - let carry9: i64 - - carry0 = (h0 + (1 << 25)) >> 26 - h1 += carry0 - h0 -= carry0 << 26 - carry4 = (h4 + (1 << 25)) >> 26 - h5 += carry4 - h4 -= carry4 << 26 - - carry1 = (h1 + (1 << 24)) >> 25 - h2 += carry1 - h1 -= carry1 << 25 - carry5 = (h5 + (1 << 24)) >> 25 - h6 += carry5 - h5 -= carry5 << 25 - - carry2 = (h2 + (1 << 25)) >> 26 - h3 += carry2 - h2 -= carry2 << 26 - carry6 = (h6 + (1 << 25)) >> 26 - h7 += carry6 - h6 -= carry6 << 26 - - carry3 = (h3 + (1 << 24)) >> 25 - h4 += carry3 - h3 -= carry3 << 25 - carry7 = (h7 + (1 << 24)) >> 25 - h8 += carry7 - h7 -= carry7 << 25 - - carry4 = (h4 + (1 << 25)) >> 26 - h5 += carry4 - h4 -= carry4 << 26 - carry8 = (h8 + (1 << 25)) >> 26 - h9 += carry8 - h8 -= carry8 << 26 - - carry9 = (h9 + (1 << 24)) >> 25 - h0 += carry9 * 19 - h9 -= carry9 << 25 - - carry0 = (h0 + (1 << 25)) >> 26 - h1 += carry0 - h0 -= carry0 << 26 - - // assign results to output - const h_ptr: usize = changetype(h) - store(h_ptr, h0, 0) - store(h_ptr, h1, 4) - store(h_ptr, h2, 8) - store(h_ptr, h3, 12) - store(h_ptr, h4, 16) - store(h_ptr, h5, 20) - store(h_ptr, h6, 24) - store(h_ptr, h7, 28) - store(h_ptr, h8, 32) - store(h_ptr, h9, 36) + // sum related accumulators and shuffle into ref10 carry schedule + let h04_sum = v128.add(v128.shuffle(h01, h45, 0, 2), v128.shuffle(h90, h34, 1, 3)) + let h15_sum = v128.add(v128.shuffle(h12, h56, 0, 2), v128.shuffle(h01, h45, 1, 3)) + let h26_sum = v128.add(v128.shuffle(h23, h67, 0, 2), v128.shuffle(h12, h56, 1, 3)) + let h37_sum = v128.add(v128.shuffle(h34, h78, 0, 2), v128.shuffle(h23, h67, 1, 3)) + let h89_sum = v128.add(v128.shuffle(h89, h90, 0, 2), v128.shuffle(h78, h89, 1, 3)) + + const add24 = v128.splat(1 << 24) + const add25 = v128.splat(1 << 25) + let carry: v128 + + carry = v128.shr(v128.add(h04_sum, add25), 26) + h04_sum = v128.sub(h04_sum, v128.shl(carry, 26)) + h15_sum = v128.add(h15_sum, carry) + + carry = v128.shr(v128.add(h15_sum, add24), 25) + h15_sum = v128.sub(h15_sum, v128.shl(carry, 25)) + h26_sum = v128.add(h26_sum, carry) + + carry = v128.shr(v128.add(h26_sum, add25), 26) + h26_sum = v128.sub(h26_sum, v128.shl(carry, 26)) + h37_sum = v128.add(h37_sum, carry) + + carry = v128.shr(v128.add(h37_sum, add24), 25) + h37_sum = v128.sub(h37_sum, v128.shl(carry, 25)) + let h48_sum = v128.add(v128.shuffle(h04_sum, h89_sum, 1, 2), carry) + + carry = v128.shr(v128.add(h48_sum, add25), 26) + h48_sum = v128.sub(h48_sum, v128.shl(carry, 26)) + let h59_sum = v128.add(v128.shuffle(h15_sum, h89_sum, 1, 3), carry) + + carry = v128.shr(v128.add(h59_sum, add24), 25) + h59_sum = v128.sub(h59_sum, v128.shl(carry, 25)) + // wrap requires multiplying carry from h9 by 19 + let h60_sum = v128.add(v128.shuffle(h26_sum, h04_sum, 1, 2), v128.mul(carry, i64x2(1, 19))) + + carry = v128.shr(v128.add(h60_sum, add25), 26) + h60_sum = v128.sub(h60_sum, v128.shl(carry, 26)) + let h71_sum = v128.add(v128.shuffle(h37_sum, h15_sum, 1, 2), carry) + + // assign scalar results to output + v128.store_lane(h_ptr, h60_sum, 2, 0) + v128.store_lane(h_ptr, h71_sum, 2, 4) + v128.store_lane(h_ptr, h26_sum, 0, 8) + v128.store_lane(h_ptr, h37_sum, 0, 12) + v128.store_lane(h_ptr, h48_sum, 0, 16) + v128.store_lane(h_ptr, h59_sum, 0, 20) + v128.store_lane(h_ptr, h60_sum, 0, 24) + v128.store_lane(h_ptr, h71_sum, 0, 28) + v128.store_lane(h_ptr, h48_sum, 2, 32) + v128.store_lane(h_ptr, h59_sum, 2, 36) } /**