Migrate multiply carry chain from scalar extraction and carry to direct vector manipu...

diff --git a/src/assembly/ed25519/fe.ts b/src/assembly/ed25519/fe.ts
index ff29d9941bbe2879a76d9781b92699409b6e438d..3ad204b99ef97cdc9761cc3bdc0411cf8723c28b 100644 (file)
--- a/src/assembly/ed25519/fe.ts
+++ b/src/assembly/ed25519/fe.ts
@@ -284,6 +284,7 @@ export function fe_iszero (f: FieldElement): u8 {
  * @param g FieldElement multiplicand source
  */
 export function fe_mul (h: FieldElement, f: FieldElement, g: FieldElement): void {
+       const h_ptr: usize = changetype<usize>(h)
        const f_ptr: usize = changetype<usize>(f)
        const g_ptr: usize = changetype<usize>(g)
        const g0123: v128 = v128.load(g_ptr, 0)
@@ -297,10 +298,9 @@ export function fe_mul (h: FieldElement, f: FieldElement, g: FieldElement): void
        const odds: v128 = i32x4(0, -1, 0, -1)
 
        // factor of 19 for wrap from h9 to h0
-       const v19: v128 = i32x4.splat(19)
-       const g0123_19: v128 = i32x4.mul(g0123, v19)
-       const g4567_19: v128 = i32x4.mul(g4567, v19)
-       const g89xx_19: v128 = i32x4.mul(g89xx, v19)
+       const g0123_19: v128 = i32x4.mul(g0123, i32x4.splat(19))
+       const g4567_19: v128 = i32x4.mul(g4567, i32x4.splat(19))
+       const g89xx_19: v128 = i32x4.mul(g89xx, i32x4.splat(19))
 
        // f[0]
        let f_i: v128 = v128.load_splat<i32>(f_ptr, 0 << 2)
@@ -313,7 +313,7 @@ export function fe_mul (h: FieldElement, f: FieldElement, g: FieldElement): void
 
        // f[1]
        f_i = v128.load_splat<i32>(f_ptr, 1 << 2)
-       f_i = i32x4.add(f_i, v128.and(f_i, odds))
+       f_i = v128.add<i32>(f_i, v128.and(f_i, odds))
 
        let h12: v128 = i64x2.extmul_low_i32x4_s(f_i, g0123)
        let h34: v128 = i64x2.extmul_high_i32x4_s(f_i, g0123)
@@ -338,7 +338,7 @@ export function fe_mul (h: FieldElement, f: FieldElement, g: FieldElement): void
 
        // f[3]
        f_i = v128.load_splat<i32>(f_ptr, 3 << 2)
-       f_i = i32x4.add(f_i, v128.and(f_i, odds))
+       f_i = v128.add<i32>(f_i, v128.and(f_i, odds))
 
        t0 = i64x2.extmul_low_i32x4_s(f_i, g0123)
        t1 = i64x2.extmul_high_i32x4_s(f_i, g0123)
@@ -369,7 +369,7 @@ export function fe_mul (h: FieldElement, f: FieldElement, g: FieldElement): void
 
        // f[5]
        f_i = v128.load_splat<i32>(f_ptr, 5 << 2)
-       f_i = i32x4.add(f_i, v128.and(f_i, odds))
+       f_i = v128.add<i32>(f_i, v128.and(f_i, odds))
 
        t0 = i64x2.extmul_low_i32x4_s(f_i, g0123)
        t1 = i64x2.extmul_high_i32x4_s(f_i, g0123)
@@ -400,7 +400,7 @@ export function fe_mul (h: FieldElement, f: FieldElement, g: FieldElement): void
 
        // f[7]
        f_i = v128.load_splat<i32>(f_ptr, 7 << 2)
-       f_i = i32x4.add(f_i, v128.and(f_i, odds))
+       f_i = v128.add<i32>(f_i, v128.and(f_i, odds))
 
        t0 = i64x2.extmul_low_i32x4_s(f_i, g0123)
        t1 = i64x2.extmul_high_i32x4_s(f_i, v128.bitselect(g0123, g0123_19, m))
@@ -431,7 +431,7 @@ export function fe_mul (h: FieldElement, f: FieldElement, g: FieldElement): void
 
        // f[9]
        f_i = v128.load_splat<i32>(f_ptr, 9 << 2)
-       f_i = i32x4.add(f_i, v128.and(f_i, odds))
+       f_i = v128.add<i32>(f_i, v128.and(f_i, odds))
 
        t0 = i64x2.extmul_low_i32x4_s(f_i, v128.bitselect(g0123, g0123_19, m))
        t1 = i64x2.extmul_high_i32x4_s(f_i, g0123_19)
@@ -445,85 +445,57 @@ export function fe_mul (h: FieldElement, f: FieldElement, g: FieldElement): void
        h56 = i64x2.add(h56, t3)
        h78 = i64x2.add(h78, t4)
 
-       // extract scalars from vectors
-       let h0: i64 = i64x2.extract_lane(h90, 1) + i64x2.extract_lane(h01, 0)
-       let h1: i64 = i64x2.extract_lane(h01, 1) + i64x2.extract_lane(h12, 0)
-       let h2: i64 = i64x2.extract_lane(h12, 1) + i64x2.extract_lane(h23, 0)
-       let h3: i64 = i64x2.extract_lane(h23, 1) + i64x2.extract_lane(h34, 0)
-       let h4: i64 = i64x2.extract_lane(h34, 1) + i64x2.extract_lane(h45, 0)
-       let h5: i64 = i64x2.extract_lane(h45, 1) + i64x2.extract_lane(h56, 0)
-       let h6: i64 = i64x2.extract_lane(h56, 1) + i64x2.extract_lane(h67, 0)
-       let h7: i64 = i64x2.extract_lane(h67, 1) + i64x2.extract_lane(h78, 0)
-       let h8: i64 = i64x2.extract_lane(h78, 1) + i64x2.extract_lane(h89, 0)
-       let h9: i64 = i64x2.extract_lane(h89, 1) + i64x2.extract_lane(h90, 0)
-
-       // reduce scalars with carry
-       let carry0: i64
-       let carry1: i64
-       let carry2: i64
-       let carry3: i64
-       let carry4: i64
-       let carry5: i64
-       let carry6: i64
-       let carry7: i64
-       let carry8: i64
-       let carry9: i64
-
-       carry0 = (h0 + (1 << 25)) >> 26
-       h1 += carry0
-       h0 -= carry0 << 26
-       carry4 = (h4 + (1 << 25)) >> 26
-       h5 += carry4
-       h4 -= carry4 << 26
-
-       carry1 = (h1 + (1 << 24)) >> 25
-       h2 += carry1
-       h1 -= carry1 << 25
-       carry5 = (h5 + (1 << 24)) >> 25
-       h6 += carry5
-       h5 -= carry5 << 25
-
-       carry2 = (h2 + (1 << 25)) >> 26
-       h3 += carry2
-       h2 -= carry2 << 26
-       carry6 = (h6 + (1 << 25)) >> 26
-       h7 += carry6
-       h6 -= carry6 << 26
-
-       carry3 = (h3 + (1 << 24)) >> 25
-       h4 += carry3
-       h3 -= carry3 << 25
-       carry7 = (h7 + (1 << 24)) >> 25
-       h8 += carry7
-       h7 -= carry7 << 25
-
-       carry4 = (h4 + (1 << 25)) >> 26
-       h5 += carry4
-       h4 -= carry4 << 26
-       carry8 = (h8 + (1 << 25)) >> 26
-       h9 += carry8
-       h8 -= carry8 << 26
-
-       carry9 = (h9 + (1 << 24)) >> 25
-       h0 += carry9 * 19
-       h9 -= carry9 << 25
-
-       carry0 = (h0 + (1 << 25)) >> 26
-       h1 += carry0
-       h0 -= carry0 << 26
-
-       // assign results to output
-       const h_ptr: usize = changetype<usize>(h)
-       store<i32>(h_ptr, h0, 0)
-       store<i32>(h_ptr, h1, 4)
-       store<i32>(h_ptr, h2, 8)
-       store<i32>(h_ptr, h3, 12)
-       store<i32>(h_ptr, h4, 16)
-       store<i32>(h_ptr, h5, 20)
-       store<i32>(h_ptr, h6, 24)
-       store<i32>(h_ptr, h7, 28)
-       store<i32>(h_ptr, h8, 32)
-       store<i32>(h_ptr, h9, 36)
+       // sum related accumulators and shuffle into ref10 carry schedule
+       let h04_sum = v128.add<i64>(v128.shuffle<i64>(h01, h45, 0, 2), v128.shuffle<i64>(h90, h34, 1, 3))
+       let h15_sum = v128.add<i64>(v128.shuffle<i64>(h12, h56, 0, 2), v128.shuffle<i64>(h01, h45, 1, 3))
+       let h26_sum = v128.add<i64>(v128.shuffle<i64>(h23, h67, 0, 2), v128.shuffle<i64>(h12, h56, 1, 3))
+       let h37_sum = v128.add<i64>(v128.shuffle<i64>(h34, h78, 0, 2), v128.shuffle<i64>(h23, h67, 1, 3))
+       let h89_sum = v128.add<i64>(v128.shuffle<i64>(h89, h90, 0, 2), v128.shuffle<i64>(h78, h89, 1, 3))
+
+       const add24 = v128.splat<i64>(1 << 24)
+       const add25 = v128.splat<i64>(1 << 25)
+       let carry: v128
+
+       carry = v128.shr<i64>(v128.add<i64>(h04_sum, add25), 26)
+       h04_sum = v128.sub<i64>(h04_sum, v128.shl<i64>(carry, 26))
+       h15_sum = v128.add<i64>(h15_sum, carry)
+
+       carry = v128.shr<i64>(v128.add<i64>(h15_sum, add24), 25)
+       h15_sum = v128.sub<i64>(h15_sum, v128.shl<i64>(carry, 25))
+       h26_sum = v128.add<i64>(h26_sum, carry)
+
+       carry = v128.shr<i64>(v128.add<i64>(h26_sum, add25), 26)
+       h26_sum = v128.sub<i64>(h26_sum, v128.shl<i64>(carry, 26))
+       h37_sum = v128.add<i64>(h37_sum, carry)
+
+       carry = v128.shr<i64>(v128.add<i64>(h37_sum, add24), 25)
+       h37_sum = v128.sub<i64>(h37_sum, v128.shl<i64>(carry, 25))
+       let h48_sum = v128.add<i64>(v128.shuffle<i64>(h04_sum, h89_sum, 1, 2), carry)
+
+       carry = v128.shr<i64>(v128.add<i64>(h48_sum, add25), 26)
+       h48_sum = v128.sub<i64>(h48_sum, v128.shl<i64>(carry, 26))
+       let h59_sum = v128.add<i64>(v128.shuffle<i64>(h15_sum, h89_sum, 1, 3), carry)
+
+       carry = v128.shr<i64>(v128.add<i64>(h59_sum, add24), 25)
+       h59_sum = v128.sub<i64>(h59_sum, v128.shl<i64>(carry, 25))
+       // wrap requires multiplying carry from h9 by 19
+       let h60_sum = v128.add<i64>(v128.shuffle<i64>(h26_sum, h04_sum, 1, 2), v128.mul<i64>(carry, i64x2(1, 19)))
+
+       carry = v128.shr<i64>(v128.add<i64>(h60_sum, add25), 26)
+       h60_sum = v128.sub<i64>(h60_sum, v128.shl<i64>(carry, 26))
+       let h71_sum = v128.add<i64>(v128.shuffle<i64>(h37_sum, h15_sum, 1, 2), carry)
+
+       // assign scalar results to output
+       v128.store_lane<i32>(h_ptr, h60_sum, 2, 0)
+       v128.store_lane<i32>(h_ptr, h71_sum, 2, 4)
+       v128.store_lane<i32>(h_ptr, h26_sum, 0, 8)
+       v128.store_lane<i32>(h_ptr, h37_sum, 0, 12)
+       v128.store_lane<i32>(h_ptr, h48_sum, 0, 16)
+       v128.store_lane<i32>(h_ptr, h59_sum, 0, 20)
+       v128.store_lane<i32>(h_ptr, h60_sum, 0, 24)
+       v128.store_lane<i32>(h_ptr, h71_sum, 0, 28)
+       v128.store_lane<i32>(h_ptr, h48_sum, 2, 32)
+       v128.store_lane<i32>(h_ptr, h59_sum, 2, 36)
 }
 
 /**