From 0485c275e565b983e1f708eb01574986fb71e316 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Tue, 27 Oct 2026 12:40:28 +0100 Subject: [PATCH 1/2] perf(gemm): read a row-major A in place (BLIS "sup" kernels) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Small products spent a fifth of their time packing A. A row-major A (unit column stride, positive row stride) is now read in place by unpacked-A kernels, the "rv" gemmsup kernels of BLIS (kernels/haswell/3/sup/bli_gemmsup_rv_haswell_asm_s6x16m.c). B stays packed, so the last panel is still zero-padded, and there is one kernel per tile height, so the bottom rows of a band never read past A's last row. - amd64 (FMA): gemmUAx16FMA (float32) and gemmUAx8FMA (float64), mr 1..6. Each A value is broadcast straight from its row through a scaled index. - arm64: gemmUAx8S (mr 1..8) and gemmUAx8D (mr 1..4). One vector load takes 4 floats (2 doubles) of a row, and the following k steps read them by lane with the by-element FMLA. A first version using load-and-replicate (VLD1R) was 26-78% SLOWER on a Neoverse N1 and was replaced. abbench, 10 rounds, one core, against v0.26.1 (64²/256²/1024²): Zen 3 float32 -28/-9.5/-3.9%, float64 -17/-5.5/~ N1 float32 -20/-7.7/~, float64 -12/-4.5/~ Tests: every GEMM case runs on both A paths (unpackA), plus an A view with an offset and lda != k, for float32 and float64. The new kernels are fenced for every height and two lda values. The SSE2 fallback test pins the packed path. Co-Authored-By: Claude Opus 5.5 --- internal/kernels/asmgen/amd64/gen.go | 89 ++ internal/kernels/asmgen/arm64/gen.go | 117 +++ internal/kernels/fence_test.go | 29 + internal/kernels/gemm.go | 48 +- internal/kernels/gemm32_amd64.go | 33 +- internal/kernels/gemm32_arm64.go | 39 +- internal/kernels/gemm32_test.go | 31 +- internal/kernels/gemm_amd64.go | 42 + internal/kernels/gemm_amd64_test.go | 9 +- internal/kernels/gemm_arm64.go | 27 + internal/kernels/gemm_test.go | 24 + internal/kernels/gemmua_other.go | 6 + internal/kernels/sum_amd64.s | 822 ++++++++++++++++ internal/kernels/sum_arm64.s | 1296 ++++++++++++++++++++++++++ 14 files changed, 2596 insertions(+), 16 deletions(-) create mode 100644 internal/kernels/gemmua_other.go diff --git a/internal/kernels/asmgen/amd64/gen.go b/internal/kernels/asmgen/amd64/gen.go index ba9d65b..9c46f24 100644 --- a/internal/kernels/asmgen/amd64/gen.go +++ b/internal/kernels/asmgen/amd64/gen.go @@ -47,6 +47,10 @@ func main() { f.Add(binKernel("divSSE2", "DIVPD", "DIVSD")) f.Add(gemmFMAKernel()) f.Add(gemm32FMAKernel()) + for mr := 1; mr <= gemmMR; mr++ { + f.Add(gemm32UAKernel(mr)) + f.Add(gemmUAKernel(mr, false)) + } f.Add(dotFMAKernel()) f.Add(gemmSSE2HalfKernel()) f.Add(amd64.FeatureProbe("hasFMA", amd64.FMA)) @@ -549,6 +553,91 @@ func gemm32FMAKernel() *emit.Function { return b.Func() } +// gemm32UAKernel builds gemmUAx16FMA(kc int, a *float32, lda int, pb, c +// *float32, ldc int): the 6x16 float32 tile with A read in place instead of +// packed, for mr rows (1..6): +// +// for p in [0,kc): C[r][col] += a[r*lda+p] * pb[p*16+col] (rx8FMA: the same loop in PD instructions, 8-byte steps. +func gemm32UAKernel(mr int) *emit.Function { return gemmUAKernel(mr, true) } + +func gemmUAKernel(mr int, single bool) *emit.Function { + sig := amd64.Layout( + []string{"kc", "a", "lda", "pb", "c", "ldc"}, + []amd64.Type{amd64.Int64, amd64.Ptr, amd64.Int64, amd64.Ptr, amd64.Ptr, amd64.Int64}, + nil, nil, + ) + name, size, shift, sfx, bcast := fmt.Sprintf("gemmUA%dx8FMA", mr), 8, 3, "PD", "VBROADCASTSD" + if single { + name, size, shift, sfx, bcast = fmt.Sprintf("gemmUA%dx16FMA", mr), 4, 2, "PS", "VBROADCASTSS" + } + b := amd64.NewFunc(name, sig, 0) + b.LoadArg("kc", "CX") + b.LoadArg("a", "SI") + b.LoadArg("lda", "R8") + b.LoadArg("pb", "DI") + b.LoadArg("c", "DX") + b.LoadArg("ldc", "R9") + b.Raw(fmt.Sprintf("SHLQ $%d, R8", shift)) // lda -> bytes + b.Raw(fmt.Sprintf("SHLQ $%d, R9", shift)) // ldc -> bytes + b.Raw("LEAQ (R8)(R8*2), R11") // 3*lda + b.Raw("LEAQ (R8)(R8*4), R12") // 5*lda + b.Raw("MOVQ DX, R10") + for r := 0; r < mr; r++ { + b.Raw("PREFETCHT0 (R10)") + b.Raw("PREFETCHT0 63(R10)") + if r < mr-1 { + b.Raw("ADDQ R9, R10") + } + } + for r := 0; r < 2*mr; r++ { + b.Raw(fmt.Sprintf("VXOR%s Y%d, Y%d, Y%d", sfx, r, r, r)) + } + b.Raw("TESTQ CX, CX") + b.Raw("JZ uastore") + b.Raw("uakloop:") + b.Raw(fmt.Sprintf("VMOVU%s (DI), Y12", sfx)) + b.Raw(fmt.Sprintf("VMOVU%s 32(DI), Y13", sfx)) + for r := 0; r < mr; r++ { + b.Raw(fmt.Sprintf("%s %s, Y14", bcast, uaRow(r))) + b.Raw(fmt.Sprintf("VFMADD231%s Y12, Y14, Y%d", sfx, 2*r)) + b.Raw(fmt.Sprintf("VFMADD231%s Y13, Y14, Y%d", sfx, 2*r+1)) + } + b.Raw(fmt.Sprintf("ADDQ $%d, SI", size)) + b.Raw("ADDQ $64, DI") // one packed B row: 16 floats or 8 doubles + b.Raw("DECQ CX") + b.Raw("JNZ uakloop") + b.Raw("uastore:") + for r := 0; r < mr; r++ { + b.Raw(fmt.Sprintf("VADD%s (DX), Y%d, Y15", sfx, 2*r)) + b.Raw(fmt.Sprintf("VMOVU%s Y15, (DX)", sfx)) + b.Raw(fmt.Sprintf("VADD%s 32(DX), Y%d, Y15", sfx, 2*r+1)) + b.Raw(fmt.Sprintf("VMOVU%s Y15, 32(DX)", sfx)) + if r < mr-1 { + b.Raw("ADDQ R9, DX") + } + } + b.Raw("VZEROUPPER") + b.Ret() + return b.Func() +} + +// uaRow is the address of row r of the unpacked A column at SI, rows lda +// bytes apart (R8), 3*lda in R11 and 5*lda in R12. +func uaRow(r int) string { + return [...]string{"(SI)", "(SI)(R8*1)", "(SI)(R8*2)", "(SI)(R11*1)", "(SI)(R8*4)", "(SI)(R12*1)"}[r] +} + // fma32Step emits one k step of the 6x16 float32 FMA tile. func fma32Step(b *amd64.Builder, step int) { ao, bo := 4*gemmMR*step, 4*gemm32NR*step diff --git a/internal/kernels/asmgen/arm64/gen.go b/internal/kernels/asmgen/arm64/gen.go index ab69ed8..479765b 100644 --- a/internal/kernels/asmgen/arm64/gen.go +++ b/internal/kernels/asmgen/arm64/gen.go @@ -26,6 +26,7 @@ package main import ( "fmt" "os" + "strings" "github.com/go-asmgen/asmgen/arm64" "github.com/go-asmgen/asmgen/emit" @@ -58,6 +59,12 @@ func main() { f.Add(extremum32Kernel("min32NEON", "VFMIN", "FMINS")) f.Add(gemmKernel()) f.Add(gemm32Kernel()) + for mr := 1; mr <= 8; mr++ { + f.Add(gemmUAKernel(mr, true)) + if mr <= 4 { + f.Add(gemmUAKernel(mr, false)) + } + } f.Add(blocks8Kernel()) if err := os.WriteFile("sum_arm64.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) @@ -471,6 +478,116 @@ func gemmKernel() *emit.Function { return b.Func() } +// gemmUAKernel builds the unpacked-A tiles (the "rv" sup kernels of BLIS, +// bli_gemmsup_rv_*): A is read in place, mr rows lda elements apart, instead +// of from a packed panel; B stays packed. +// +// single: gemmUAx8S(kc int, a *float32, lda int, pb, c *float32, ldc int), mr <= 8 +// double: gemmUAx8D(kc int, a *float64, lda int, pb, c *float64, ldc int), mr <= 4 +// +// computing C[r][col] += a[r*lda+p] * pb[p*8+col] for r < mr. Each row has its +// own pointer (R5..R12). A row is contiguous along k, so one plain vector load +// brings U consecutive k values of it (U = 4 floats or 2 doubles) into V20+r, +// and the U k steps that follow take their A value from lane s of that vector +// with the by-element FMLA (fmlaElem/fmlaElemS), as the packed kernels do. +// The kc%U remaining steps load each row's value as a scalar into lane 0. +// A first version loaded every A value with a load-and-replicate (VLD1R) and +// used the vector FMLA: on a Neoverse N1 it was 26-78% slower than packing A, +// apparently because the replicating loads compete with the FMAs for the +// vector pipes. +// Accumulators: float32 row r in V(2r), V(2r+1); float64 row r in V(4r)..V(4r+3). +func gemmUAKernel(mr int, single bool) *emit.Function { + sig := arm64.Layout( + []string{"kc", "a", "lda", "pb", "c", "ldc"}, + []arm64.Type{arm64.Int64, arm64.Ptr, arm64.Int64, arm64.Ptr, arm64.Ptr, arm64.Int64}, + nil, nil, + ) + name, shift, arr, vecs := fmt.Sprintf("gemmUA%dx8D", mr), 3, "D2", 4 + if single { + name, shift, arr, vecs = fmt.Sprintf("gemmUA%dx8S", mr), 2, "S4", 2 + } + size := 1 << shift + bregs := make([]string, vecs) + for v := range bregs { + bregs[v] = fmt.Sprintf("V%d.%s", 16+v, arr) + } + blist := "[" + strings.Join(bregs, ", ") + "]" + b := arm64.NewFunc(name, sig, 0) + b.LoadArg("kc", "R0"). + LoadArg("a", "R1"). + LoadArg("lda", "R2"). + LoadArg("pb", "R13"). + LoadArg("c", "R3"). + LoadArg("ldc", "R4"). + Raw(fmt.Sprintf("LSL $%d, R2, R2", shift)). + Raw(fmt.Sprintf("LSL $%d, R4, R4", shift)) + for r := 0; r < mr; r++ { // row pointers R5.. = a + r*lda + if r == 0 { + b.Raw("MOVD R1, R5") + } else { + b.Raw(fmt.Sprintf("ADD R2, R%d, R%d", 4+r, 5+r)) + } + } + for v := 0; v < mr*vecs; v++ { + b.Raw(fmt.Sprintf("VEOR V%d.B16, V%d.B16, V%d.B16", v, v, v)) + } + u := 16 / size // k steps per A vector + fmla, scalar := fmlaElem, "FMOVD" + if single { + fmla, scalar = fmlaElemS, "FMOVS" + } + step := func(lane int) { + b.Raw(fmt.Sprintf("VLD1.P %d(R13), %s", 8*size, blist)) + for r := 0; r < mr; r++ { + for v := 0; v < vecs; v++ { + b.Raw(fmla(vecs*r+v, 16+v, 20+r, lane)) + } + } + } + b.Raw(fmt.Sprintf("LSR $%d, R0, R14", shift4(u))). // whole groups of u steps + Raw(fmt.Sprintf("AND $%d, R0, R15", u-1)). // remaining steps + Raw("CBZ R14, uatail"). + Raw("uakloop:") + for r := 0; r < mr; r++ { + b.Raw(fmt.Sprintf("VLD1.P 16(R%d), [V%d.%s]", 5+r, 20+r, arr)) + } + for s := 0; s < u; s++ { + step(s) + } + b.Raw("SUB $1, R14"). + Raw("CBNZ R14, uakloop"). + Raw("uatail:"). + Raw("CBZ R15, uastore"). + Raw("uatloop:") + for r := 0; r < mr; r++ { + b.Raw(fmt.Sprintf("%s.P %d(R%d), F%d", scalar, size, 5+r, 20+r)) + } + step(0) + b.Raw("SUB $1, R15"). + Raw("CBNZ R15, uatloop"). + Raw("uastore:") + for r := 0; r < mr; r++ { + b.Raw(fmt.Sprintf("VLD1 (R3), %s", blist)) + for v := 0; v < vecs; v++ { + b.Raw(fmt.Sprintf("VFADD V%d.%s, V%d.%s, V%d.%s", vecs*r+v, arr, 16+v, arr, 16+v, arr)) + } + b.Raw(fmt.Sprintf("VST1 %s, (R3)", blist)) + if r < mr-1 { + b.Raw("ADD R4, R3, R3") + } + } + b.Ret() + return b.Func() +} + +// shift4 is log2 of u, 2 or 4. +func shift4(u int) int { + if u == 4 { + return 2 + } + return 1 +} + // fmlaElemS is fmlaElem for single precision: // // FMLA V{acc}.4S, V{breg}.4S, V{areg}.S[idx] => V{acc} += V{breg} * V{areg}[idx] diff --git a/internal/kernels/fence_test.go b/internal/kernels/fence_test.go index 3ec20e9..2bbcbff 100644 --- a/internal/kernels/fence_test.go +++ b/internal/kernels/fence_test.go @@ -134,6 +134,23 @@ func TestGemmMicroStaysInsideItsOperands(t *testing.T) { } run() forEachFallback(run) + if gemm64.microUA == nil { + return + } + for kc := 1; kc <= 9; kc++ { + for _, ldc := range []int{NR, NR + 3} { + pb := fences(t, kc*NR)[1] + for mr := 1; mr <= MR; mr++ { + for _, lda := range []int{kc, kc + 5} { + for _, a := range fences(t, (mr-1)*lda+kc) { + for _, dst := range fences(t, (mr-1)*ldc+NR) { + gemm64.microUA(kc, a, lda, pb, dst, ldc, mr) + } + } + } + } + } + } } // TestGemmMicro32StaysInsideItsOperands fences the float32 GEMM kernel's @@ -150,6 +167,18 @@ func TestGemmMicro32StaysInsideItsOperands(t *testing.T) { for _, dst := range fences32(t, (g.mr-1)*ldc+g.nr) { g.micro(kc, pa, pb, dst, ldc) } + if g.microUA == nil { + continue + } + for mr := 1; mr <= g.mr; mr++ { + for _, lda := range []int{kc, kc + 5} { + for _, a := range fences32(t, (mr-1)*lda+kc) { + for _, dst := range fences32(t, (mr-1)*ldc+g.nr) { + g.microUA(kc, a, lda, pb, dst, ldc, mr) + } + } + } + } } } } diff --git a/internal/kernels/gemm.go b/internal/kernels/gemm.go index efe62e4..6ee79d4 100644 --- a/internal/kernels/gemm.go +++ b/internal/kernels/gemm.go @@ -81,14 +81,23 @@ func rowMajor[T gemmFloat](data []T, ld int) MatOf[T] { // gemmKernel is one element type's micro-kernel, the mr x nr tile it computes, // and the pools of pack buffers sized for that tile. micro adds the mr x nr tile // sum_p pa[p*mr..]*pb[p*nr..] into dst, whose rows are ldc apart; see gemmMicro. +// +// microUA, where there is one, is the same tile with A read in place: rows +// lda > 0 elements apart from a[0], k steps along each row, for the top mr +// (1..mr) rows only. gemmBand then never packs a row-major A. type gemmKernel[T gemmFloat] struct { mr, nr int micro func(kc int, pa, pb, dst []T, ldc int) + microUA func(kc int, a []T, lda int, pb, dst []T, ldc, mr int) paPool, pbPool sync.Pool } +// unpackA lets the GEMM read a row-major A in place where the kernel can. It +// is a var so tests can drive the packed path on the same inputs. +var unpackA = true + // gemm64 is the float64 GEMM, on the per-architecture gemmMicro. -var gemm64 = &gemmKernel[float64]{mr: MR, nr: NR, micro: gemmMicro} +var gemm64 = &gemmKernel[float64]{mr: MR, nr: NR, micro: gemmMicro, microUA: gemmMicroUA} // packBuf is a pair of pack buffers: pa for the MC*KC A panel, pb for the KC*NC // B panel. The two come from separate pools, because the parallel GEMM needs @@ -173,7 +182,11 @@ func (g *gemmKernel[T]) packGemmRows(r0, r1 int, dst []T, a, b MatOf[T], k, n in // dominant non-scaling cost of the old row-split parallelism. func (g *gemmKernel[T]) gemmBand(r0, r1, jc, nc, pc, kc int, dst []T, a MatOf[T], n int, pa, pb []T) { mr0, nr0 := g.mr, g.nr - tile := pa[len(pa)-mr0*nr0:] // see paCap + tile := pa[len(pa)-mr0*nr0:] // see paCap + if g.microUA != nil && unpackA && a.CS == 1 && a.RS > 0 { + g.gemmBandUA(r0, r1, jc, nc, pc, kc, dst, a, n, pb, tile) + return + } for ic := r0; ic < r1; ic += blockMC { // L1 row block of A mc := min(blockMC, r1-ic) g.packA(a, ic, mc, pc, kc, pa) // pack A(mc x kc) -> MR-tall panels @@ -192,6 +205,37 @@ func (g *gemmKernel[T]) gemmBand(r0, r1, jc, nc, pc, kc int, dst []T, a MatOf[T] } } +// gemmBandUA is gemmBand for a row-major A read in place by microUA: no A +// panel is packed. Each tile's A is the mr rows starting at row ic+ir, column +// pc, a.RS apart. A tile narrower than NR (the last B panel, zero-padded) runs +// into the scratch tile and only its nr columns are added into dst; a tile +// shorter than MR uses the kernel for its own height, so no row past A's last +// is read. +func (g *gemmKernel[T]) gemmBandUA(r0, r1, jc, nc, pc, kc int, dst []T, a MatOf[T], n int, pb, tile []T) { + mr0, nr0 := g.mr, g.nr + for jr := 0; jr < nc; jr += nr0 { + nr := min(nr0, nc-jr) + bp := pb[jr*kc:] + for i := r0; i < r1; i += mr0 { + mr := min(mr0, r1-i) + ar := a.Data[a.Off+i*a.RS+pc:] + c0 := i*n + jc + jr + if nr == nr0 { + g.microUA(kc, ar, a.RS, bp, dst[c0:], n, mr) + continue + } + clear(tile) + g.microUA(kc, ar, a.RS, bp, tile, nr0, mr) + for r := 0; r < mr; r++ { + row := dst[c0+r*n : c0+r*n+nr] + for c := range row { + row[c] += tile[r*nr0+c] + } + } + } + } +} + // packB copies B[pc:pc+kc][jc:jc+nc] into NR-wide column panels: for each panel // jr the layout is pb[jr*kc + p*NR + c] = B(pc+p, jc+jr+c), with the c >= nr // columns of an edge panel zero-filled so the micro-kernel reads a full NR. diff --git a/internal/kernels/gemm32_amd64.go b/internal/kernels/gemm32_amd64.go index e928375..daac694 100644 --- a/internal/kernels/gemm32_amd64.go +++ b/internal/kernels/gemm32_amd64.go @@ -9,7 +9,20 @@ func newGemm32(fma bool) *gemmKernel[float32] { if !fma { return nil } - return &gemmKernel[float32]{mr: 6, nr: 16, micro: gemmMicro32} + return &gemmKernel[float32]{mr: 6, nr: 16, micro: gemmMicro32, microUA: gemmMicroUA32} +} + +// gemmUA32 holds the unpacked-A kernels by tile height. +var gemmUA32 = [...]func(kc int, a *float32, lda int, pb, c *float32, ldc int){ + 1: gemmUA1x16FMA, 2: gemmUA2x16FMA, 3: gemmUA3x16FMA, + 4: gemmUA4x16FMA, 5: gemmUA5x16FMA, 6: gemmUA6x16FMA, +} + +// gemmMicroUA32 adds the mr x 16 tile sum_p a[r*lda+p]*pb[p*16..] into dst, +// whose rows are ldc apart, reading A in place (mr in 1..6, lda > 0). +func gemmMicroUA32(kc int, a []float32, lda int, pb, dst []float32, ldc, mr int) { + _, _, _ = a[(mr-1)*lda+kc-1], pb[kc*16-1], dst[(mr-1)*ldc+15] // the kernel's whole reach + gemmUA32[mr](kc, &a[0], lda, &pb[0], &dst[0], ldc) } // gemmMicro32 adds the 6 x 16 tile sum_p pa[p*6..]*pb[p*16..] into the C block @@ -22,3 +35,21 @@ func gemmMicro32(kc int, pa, pb, dst []float32, ldc int) { //go:noescape func gemmMicro6x16FMA(kc int, pa, pb, c *float32, ldc int) + +//go:noescape +func gemmUA1x16FMA(kc int, a *float32, lda int, pb, c *float32, ldc int) + +//go:noescape +func gemmUA2x16FMA(kc int, a *float32, lda int, pb, c *float32, ldc int) + +//go:noescape +func gemmUA3x16FMA(kc int, a *float32, lda int, pb, c *float32, ldc int) + +//go:noescape +func gemmUA4x16FMA(kc int, a *float32, lda int, pb, c *float32, ldc int) + +//go:noescape +func gemmUA5x16FMA(kc int, a *float32, lda int, pb, c *float32, ldc int) + +//go:noescape +func gemmUA6x16FMA(kc int, a *float32, lda int, pb, c *float32, ldc int) diff --git a/internal/kernels/gemm32_arm64.go b/internal/kernels/gemm32_arm64.go index 22f1a3d..b1bcde5 100644 --- a/internal/kernels/gemm32_arm64.go +++ b/internal/kernels/gemm32_arm64.go @@ -2,7 +2,44 @@ package kernels // gemm32 is the float32 GEMM on the NEON 8x8 kernel (gemmMicro8x8S in // sum_arm64.s, generated by go-asmgen; see asmgen/arm64/gen.go). -var gemm32 = &gemmKernel[float32]{mr: 8, nr: 8, micro: gemmMicro32} +var gemm32 = &gemmKernel[float32]{mr: 8, nr: 8, micro: gemmMicro32, microUA: gemmMicroUA32} + +// gemmUA32 holds the unpacked-A kernels by tile height. +var gemmUA32 = [...]func(kc int, a *float32, lda int, pb, c *float32, ldc int){ + 1: gemmUA1x8S, 2: gemmUA2x8S, 3: gemmUA3x8S, 4: gemmUA4x8S, + 5: gemmUA5x8S, 6: gemmUA6x8S, 7: gemmUA7x8S, 8: gemmUA8x8S, +} + +// gemmMicroUA32 adds the mr x 8 tile sum_p a[r*lda+p]*pb[p*8..] into dst, +// whose rows are ldc apart, reading A in place (mr in 1..8, lda > 0). +func gemmMicroUA32(kc int, a []float32, lda int, pb, dst []float32, ldc, mr int) { + _, _, _ = a[(mr-1)*lda+kc-1], pb[kc*8-1], dst[(mr-1)*ldc+7] // the kernel's whole reach + gemmUA32[mr](kc, &a[0], lda, &pb[0], &dst[0], ldc) +} + +//go:noescape +func gemmUA1x8S(kc int, a *float32, lda int, pb, c *float32, ldc int) + +//go:noescape +func gemmUA2x8S(kc int, a *float32, lda int, pb, c *float32, ldc int) + +//go:noescape +func gemmUA3x8S(kc int, a *float32, lda int, pb, c *float32, ldc int) + +//go:noescape +func gemmUA4x8S(kc int, a *float32, lda int, pb, c *float32, ldc int) + +//go:noescape +func gemmUA5x8S(kc int, a *float32, lda int, pb, c *float32, ldc int) + +//go:noescape +func gemmUA6x8S(kc int, a *float32, lda int, pb, c *float32, ldc int) + +//go:noescape +func gemmUA7x8S(kc int, a *float32, lda int, pb, c *float32, ldc int) + +//go:noescape +func gemmUA8x8S(kc int, a *float32, lda int, pb, c *float32, ldc int) // gemmMicro32 adds the 8 x 8 tile sum_p pa[p*8..]*pb[p*8..] into the C block at // dst[0:], whose rows are ldc apart. pa, pb hold at least kc*8 packed diff --git a/internal/kernels/gemm32_test.go b/internal/kernels/gemm32_test.go index 15460b5..a2202dd 100644 --- a/internal/kernels/gemm32_test.go +++ b/internal/kernels/gemm32_test.go @@ -35,16 +35,22 @@ func assertGemm32(t *testing.T, m, k, n int, a, b Mat32) { t.Helper() want := make([]float32, m*n) gemmOracle32(want, a, b, m, k, n) - for _, gemm := range []int{1 << 30, 1} { - withThresholds(1<<20, gemm, func() { - got := make([]float32, m*n) - MatMul32StridedP(got, a, b, m, k, n) - for i := range want { - if got[i] != want[i] { - t.Fatalf("m=%d k=%d n=%d gemm=%d [%d]: %v != %v", m, k, n, gemm, i, got[i], want[i]) + saved := unpackA + defer func() { unpackA = saved }() + for _, unpacked := range []bool{true, false} { + unpackA = unpacked + for _, gemm := range []int{1 << 30, 1} { + withThresholds(1<<20, gemm, func() { + got := make([]float32, m*n) + MatMul32StridedP(got, a, b, m, k, n) + for i := range want { + if got[i] != want[i] { + t.Fatalf("m=%d k=%d n=%d gemm=%d unpackA=%v [%d]: %v != %v", + m, k, n, gemm, unpacked, i, got[i], want[i]) + } } - } - }) + }) + } } } @@ -73,6 +79,13 @@ func TestGemm32Correctness(t *testing.T) { bt.Data[3+j*s.k+p] = b.Data[p*s.n+j] } } + // as holds A behind a 2-element offset, rows k+3 apart: the + // unpacked-A kernels read it through lda != k. + as := Mat32{Data: make([]float32, 2+s.m*(s.k+3)), Off: 2, RS: s.k + 3, CS: 1} + for i := 0; i < s.m; i++ { + copy(as.Data[2+i*(s.k+3):], a.Data[i*s.k:(i+1)*s.k]) + } + assertGemm32(t, s.m, s.k, s.n, as, b) for _, bv := range []Mat32{b, bt} { assertGemm32(t, s.m, s.k, s.n, a, bv) withBlocks(MR, 2, NR, func() { assertGemm32(t, s.m, s.k, s.n, a, bv) }) diff --git a/internal/kernels/gemm_amd64.go b/internal/kernels/gemm_amd64.go index 718e792..62e91d9 100644 --- a/internal/kernels/gemm_amd64.go +++ b/internal/kernels/gemm_amd64.go @@ -41,6 +41,48 @@ func gemmMicro(kc int, pa, pb, dst []float64, ldc int) { gemmMicro6x4SSE2(kc, &pa[0], &pb[4], &dst[4], ldc) } +// gemmMicroUA is the unpacked-A float64 kernel (see gemmKernel.microUA), on +// FMA CPUs only; without FMA, A is packed for the SSE2 kernel. +var gemmMicroUA = microUAIf(useFMA) + +func microUAIf(fma bool) func(kc int, a []float64, lda int, pb, dst []float64, ldc, mr int) { + if !fma { + return nil + } + return gemmMicroUA64 +} + +// gemmUA64 holds the unpacked-A kernels by tile height. +var gemmUA64 = [...]func(kc int, a *float64, lda int, pb, c *float64, ldc int){ + 1: gemmUA1x8FMA, 2: gemmUA2x8FMA, 3: gemmUA3x8FMA, + 4: gemmUA4x8FMA, 5: gemmUA5x8FMA, 6: gemmUA6x8FMA, +} + +// gemmMicroUA64 adds the mr x 8 tile sum_p a[r*lda+p]*pb[p*8..] into dst, +// whose rows are ldc apart, reading A in place (mr in 1..6, lda > 0). +func gemmMicroUA64(kc int, a []float64, lda int, pb, dst []float64, ldc, mr int) { + _, _, _ = a[(mr-1)*lda+kc-1], pb[kc*NR-1], dst[(mr-1)*ldc+NR-1] // the kernel's whole reach + gemmUA64[mr](kc, &a[0], lda, &pb[0], &dst[0], ldc) +} + +//go:noescape +func gemmUA1x8FMA(kc int, a *float64, lda int, pb, c *float64, ldc int) + +//go:noescape +func gemmUA2x8FMA(kc int, a *float64, lda int, pb, c *float64, ldc int) + +//go:noescape +func gemmUA3x8FMA(kc int, a *float64, lda int, pb, c *float64, ldc int) + +//go:noescape +func gemmUA4x8FMA(kc int, a *float64, lda int, pb, c *float64, ldc int) + +//go:noescape +func gemmUA5x8FMA(kc int, a *float64, lda int, pb, c *float64, ldc int) + +//go:noescape +func gemmUA6x8FMA(kc int, a *float64, lda int, pb, c *float64, ldc int) + //go:noescape func gemmMicro6x8FMA(kc int, pa, pb, c *float64, ldc int) diff --git a/internal/kernels/gemm_amd64_test.go b/internal/kernels/gemm_amd64_test.go index 67cfd66..7ada228 100644 --- a/internal/kernels/gemm_amd64_test.go +++ b/internal/kernels/gemm_amd64_test.go @@ -7,9 +7,12 @@ import "testing" // (or a hypervisor that masks FMA) takes. The FMA path is the default and is // exercised by the rest of the suite wherever hasFMA is true. func TestGemmSSE2Fallback(t *testing.T) { - saved := useFMA - useFMA = false - defer func() { useFMA = saved }() + saved, savedUA := useFMA, unpackA + useFMA, unpackA = false, false // the SSE2 kernel reads packed A only + defer func() { useFMA, unpackA = saved, savedUA }() + if microUAIf(false) != nil || microUAIf(true) == nil { + t.Fatal("microUAIf: an unpacked-A kernel without FMA, or none with it") + } t.Run("correctness", TestGemmCorrectness) t.Run("bands", TestGemmBandSizing) t.Run("float", TestGemmFloatTolerance) diff --git a/internal/kernels/gemm_arm64.go b/internal/kernels/gemm_arm64.go index 51402f0..406e82b 100644 --- a/internal/kernels/gemm_arm64.go +++ b/internal/kernels/gemm_arm64.go @@ -41,3 +41,30 @@ func dotNEON(a, b *float64, n int) float64 //go:noescape func gemmMicro4x8(kc int, pa, pb, c *float64, ldc int) + +// gemmMicroUA is the unpacked-A float64 kernel (see gemmKernel.microUA). +var gemmMicroUA = gemmMicroUA64 + +// gemmUA64 holds the unpacked-A kernels by tile height. +var gemmUA64 = [...]func(kc int, a *float64, lda int, pb, c *float64, ldc int){ + 1: gemmUA1x8D, 2: gemmUA2x8D, 3: gemmUA3x8D, 4: gemmUA4x8D, +} + +// gemmMicroUA64 adds the mr x 8 tile sum_p a[r*lda+p]*pb[p*8..] into dst, +// whose rows are ldc apart, reading A in place (mr in 1..4, lda > 0). +func gemmMicroUA64(kc int, a []float64, lda int, pb, dst []float64, ldc, mr int) { + _, _, _ = a[(mr-1)*lda+kc-1], pb[kc*NR-1], dst[(mr-1)*ldc+NR-1] // the kernel's whole reach + gemmUA64[mr](kc, &a[0], lda, &pb[0], &dst[0], ldc) +} + +//go:noescape +func gemmUA1x8D(kc int, a *float64, lda int, pb, c *float64, ldc int) + +//go:noescape +func gemmUA2x8D(kc int, a *float64, lda int, pb, c *float64, ldc int) + +//go:noescape +func gemmUA3x8D(kc int, a *float64, lda int, pb, c *float64, ldc int) + +//go:noescape +func gemmUA4x8D(kc int, a *float64, lda int, pb, c *float64, ldc int) diff --git a/internal/kernels/gemm_test.go b/internal/kernels/gemm_test.go index bdde8ce..1ec642c 100644 --- a/internal/kernels/gemm_test.go +++ b/internal/kernels/gemm_test.go @@ -41,6 +41,30 @@ func withBandTiles(bt int, f func()) { // the oracle bit-for-bit for the given shape, using the integer-valued operands // that make the ikj order reproduce the oracle exactly. func assertGemm(t *testing.T, m, k, n int, a, b []float64) { + t.Helper() + saved := unpackA + defer func() { unpackA = saved }() + for _, unpacked := range []bool{saved, false} { + unpackA = unpacked + assertGemmPath(t, m, k, n, a, b) + } + // A in place behind an offset, rows k+3 apart (lda != k). + as := Mat{Data: make([]float64, 2+m*(k+3)), Off: 2, RS: k + 3, CS: 1} + for i := 0; i < m; i++ { + copy(as.Data[2+i*(k+3):], a[i*k:(i+1)*k]) + } + want := make([]float64, m*n) + gemmOracle(want, a, b, m, k, n) + got := make([]float64, m*n) + MatMulStridedP(got, as, rowMajor(b, n), m, k, n) + for i := range want { + if got[i] != want[i] { + t.Fatalf("strided A m=%d k=%d n=%d [%d]: %v != %v", m, k, n, i, got[i], want[i]) + } + } +} + +func assertGemmPath(t *testing.T, m, k, n int, a, b []float64) { t.Helper() want := make([]float64, m*n) gemmOracle(want, a, b, m, k, n) diff --git a/internal/kernels/gemmua_other.go b/internal/kernels/gemmua_other.go new file mode 100644 index 0000000..190943d --- /dev/null +++ b/internal/kernels/gemmua_other.go @@ -0,0 +1,6 @@ +//go:build !amd64 && !arm64 + +package kernels + +// gemmMicroUA is nil here: no unpacked-A float64 kernel, so A is packed. +var gemmMicroUA func(kc int, a []float64, lda int, pb, dst []float64, ldc, mr int) diff --git a/internal/kernels/sum_amd64.s b/internal/kernels/sum_amd64.s index 09a353d..fa71a7e 100644 --- a/internal/kernels/sum_amd64.s +++ b/internal/kernels/sum_amd64.s @@ -677,6 +677,828 @@ TEXT ·gemmMicro6x16FMA(SB), NOSPLIT, $0-40 VZEROUPPER RET +TEXT ·gemmUA1x16FMA(SB), NOSPLIT, $0-48 + MOVQ kc+0(FP), CX + MOVQ a+8(FP), SI + MOVQ lda+16(FP), R8 + MOVQ pb+24(FP), DI + MOVQ c+32(FP), DX + MOVQ ldc+40(FP), R9 + SHLQ $2, R8 + SHLQ $2, R9 + LEAQ (R8)(R8*2), R11 + LEAQ (R8)(R8*4), R12 + MOVQ DX, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + VXORPS Y0, Y0, Y0 + VXORPS Y1, Y1, Y1 + TESTQ CX, CX + JZ uastore + uakloop: + VMOVUPS (DI), Y12 + VMOVUPS 32(DI), Y13 + VBROADCASTSS (SI), Y14 + VFMADD231PS Y12, Y14, Y0 + VFMADD231PS Y13, Y14, Y1 + ADDQ $4, SI + ADDQ $64, DI + DECQ CX + JNZ uakloop + uastore: + VADDPS (DX), Y0, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y1, Y15 + VMOVUPS Y15, 32(DX) + VZEROUPPER + RET + +TEXT ·gemmUA1x8FMA(SB), NOSPLIT, $0-48 + MOVQ kc+0(FP), CX + MOVQ a+8(FP), SI + MOVQ lda+16(FP), R8 + MOVQ pb+24(FP), DI + MOVQ c+32(FP), DX + MOVQ ldc+40(FP), R9 + SHLQ $3, R8 + SHLQ $3, R9 + LEAQ (R8)(R8*2), R11 + LEAQ (R8)(R8*4), R12 + MOVQ DX, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + VXORPD Y0, Y0, Y0 + VXORPD Y1, Y1, Y1 + TESTQ CX, CX + JZ uastore + uakloop: + VMOVUPD (DI), Y12 + VMOVUPD 32(DI), Y13 + VBROADCASTSD (SI), Y14 + VFMADD231PD Y12, Y14, Y0 + VFMADD231PD Y13, Y14, Y1 + ADDQ $8, SI + ADDQ $64, DI + DECQ CX + JNZ uakloop + uastore: + VADDPD (DX), Y0, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y1, Y15 + VMOVUPD Y15, 32(DX) + VZEROUPPER + RET + +TEXT ·gemmUA2x16FMA(SB), NOSPLIT, $0-48 + MOVQ kc+0(FP), CX + MOVQ a+8(FP), SI + MOVQ lda+16(FP), R8 + MOVQ pb+24(FP), DI + MOVQ c+32(FP), DX + MOVQ ldc+40(FP), R9 + SHLQ $2, R8 + SHLQ $2, R9 + LEAQ (R8)(R8*2), R11 + LEAQ (R8)(R8*4), R12 + MOVQ DX, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + VXORPS Y0, Y0, Y0 + VXORPS Y1, Y1, Y1 + VXORPS Y2, Y2, Y2 + VXORPS Y3, Y3, Y3 + TESTQ CX, CX + JZ uastore + uakloop: + VMOVUPS (DI), Y12 + VMOVUPS 32(DI), Y13 + VBROADCASTSS (SI), Y14 + VFMADD231PS Y12, Y14, Y0 + VFMADD231PS Y13, Y14, Y1 + VBROADCASTSS (SI)(R8*1), Y14 + VFMADD231PS Y12, Y14, Y2 + VFMADD231PS Y13, Y14, Y3 + ADDQ $4, SI + ADDQ $64, DI + DECQ CX + JNZ uakloop + uastore: + VADDPS (DX), Y0, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y1, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R9, DX + VADDPS (DX), Y2, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y3, Y15 + VMOVUPS Y15, 32(DX) + VZEROUPPER + RET + +TEXT ·gemmUA2x8FMA(SB), NOSPLIT, $0-48 + MOVQ kc+0(FP), CX + MOVQ a+8(FP), SI + MOVQ lda+16(FP), R8 + MOVQ pb+24(FP), DI + MOVQ c+32(FP), DX + MOVQ ldc+40(FP), R9 + SHLQ $3, R8 + SHLQ $3, R9 + LEAQ (R8)(R8*2), R11 + LEAQ (R8)(R8*4), R12 + MOVQ DX, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + VXORPD Y0, Y0, Y0 + VXORPD Y1, Y1, Y1 + VXORPD Y2, Y2, Y2 + VXORPD Y3, Y3, Y3 + TESTQ CX, CX + JZ uastore + uakloop: + VMOVUPD (DI), Y12 + VMOVUPD 32(DI), Y13 + VBROADCASTSD (SI), Y14 + VFMADD231PD Y12, Y14, Y0 + VFMADD231PD Y13, Y14, Y1 + VBROADCASTSD (SI)(R8*1), Y14 + VFMADD231PD Y12, Y14, Y2 + VFMADD231PD Y13, Y14, Y3 + ADDQ $8, SI + ADDQ $64, DI + DECQ CX + JNZ uakloop + uastore: + VADDPD (DX), Y0, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y1, Y15 + VMOVUPD Y15, 32(DX) + ADDQ R9, DX + VADDPD (DX), Y2, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y3, Y15 + VMOVUPD Y15, 32(DX) + VZEROUPPER + RET + +TEXT ·gemmUA3x16FMA(SB), NOSPLIT, $0-48 + MOVQ kc+0(FP), CX + MOVQ a+8(FP), SI + MOVQ lda+16(FP), R8 + MOVQ pb+24(FP), DI + MOVQ c+32(FP), DX + MOVQ ldc+40(FP), R9 + SHLQ $2, R8 + SHLQ $2, R9 + LEAQ (R8)(R8*2), R11 + LEAQ (R8)(R8*4), R12 + MOVQ DX, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + VXORPS Y0, Y0, Y0 + VXORPS Y1, Y1, Y1 + VXORPS Y2, Y2, Y2 + VXORPS Y3, Y3, Y3 + VXORPS Y4, Y4, Y4 + VXORPS Y5, Y5, Y5 + TESTQ CX, CX + JZ uastore + uakloop: + VMOVUPS (DI), Y12 + VMOVUPS 32(DI), Y13 + VBROADCASTSS (SI), Y14 + VFMADD231PS Y12, Y14, Y0 + VFMADD231PS Y13, Y14, Y1 + VBROADCASTSS (SI)(R8*1), Y14 + VFMADD231PS Y12, Y14, Y2 + VFMADD231PS Y13, Y14, Y3 + VBROADCASTSS (SI)(R8*2), Y14 + VFMADD231PS Y12, Y14, Y4 + VFMADD231PS Y13, Y14, Y5 + ADDQ $4, SI + ADDQ $64, DI + DECQ CX + JNZ uakloop + uastore: + VADDPS (DX), Y0, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y1, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R9, DX + VADDPS (DX), Y2, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y3, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R9, DX + VADDPS (DX), Y4, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y5, Y15 + VMOVUPS Y15, 32(DX) + VZEROUPPER + RET + +TEXT ·gemmUA3x8FMA(SB), NOSPLIT, $0-48 + MOVQ kc+0(FP), CX + MOVQ a+8(FP), SI + MOVQ lda+16(FP), R8 + MOVQ pb+24(FP), DI + MOVQ c+32(FP), DX + MOVQ ldc+40(FP), R9 + SHLQ $3, R8 + SHLQ $3, R9 + LEAQ (R8)(R8*2), R11 + LEAQ (R8)(R8*4), R12 + MOVQ DX, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + VXORPD Y0, Y0, Y0 + VXORPD Y1, Y1, Y1 + VXORPD Y2, Y2, Y2 + VXORPD Y3, Y3, Y3 + VXORPD Y4, Y4, Y4 + VXORPD Y5, Y5, Y5 + TESTQ CX, CX + JZ uastore + uakloop: + VMOVUPD (DI), Y12 + VMOVUPD 32(DI), Y13 + VBROADCASTSD (SI), Y14 + VFMADD231PD Y12, Y14, Y0 + VFMADD231PD Y13, Y14, Y1 + VBROADCASTSD (SI)(R8*1), Y14 + VFMADD231PD Y12, Y14, Y2 + VFMADD231PD Y13, Y14, Y3 + VBROADCASTSD (SI)(R8*2), Y14 + VFMADD231PD Y12, Y14, Y4 + VFMADD231PD Y13, Y14, Y5 + ADDQ $8, SI + ADDQ $64, DI + DECQ CX + JNZ uakloop + uastore: + VADDPD (DX), Y0, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y1, Y15 + VMOVUPD Y15, 32(DX) + ADDQ R9, DX + VADDPD (DX), Y2, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y3, Y15 + VMOVUPD Y15, 32(DX) + ADDQ R9, DX + VADDPD (DX), Y4, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y5, Y15 + VMOVUPD Y15, 32(DX) + VZEROUPPER + RET + +TEXT ·gemmUA4x16FMA(SB), NOSPLIT, $0-48 + MOVQ kc+0(FP), CX + MOVQ a+8(FP), SI + MOVQ lda+16(FP), R8 + MOVQ pb+24(FP), DI + MOVQ c+32(FP), DX + MOVQ ldc+40(FP), R9 + SHLQ $2, R8 + SHLQ $2, R9 + LEAQ (R8)(R8*2), R11 + LEAQ (R8)(R8*4), R12 + MOVQ DX, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + VXORPS Y0, Y0, Y0 + VXORPS Y1, Y1, Y1 + VXORPS Y2, Y2, Y2 + VXORPS Y3, Y3, Y3 + VXORPS Y4, Y4, Y4 + VXORPS Y5, Y5, Y5 + VXORPS Y6, Y6, Y6 + VXORPS Y7, Y7, Y7 + TESTQ CX, CX + JZ uastore + uakloop: + VMOVUPS (DI), Y12 + VMOVUPS 32(DI), Y13 + VBROADCASTSS (SI), Y14 + VFMADD231PS Y12, Y14, Y0 + VFMADD231PS Y13, Y14, Y1 + VBROADCASTSS (SI)(R8*1), Y14 + VFMADD231PS Y12, Y14, Y2 + VFMADD231PS Y13, Y14, Y3 + VBROADCASTSS (SI)(R8*2), Y14 + VFMADD231PS Y12, Y14, Y4 + VFMADD231PS Y13, Y14, Y5 + VBROADCASTSS (SI)(R11*1), Y14 + VFMADD231PS Y12, Y14, Y6 + VFMADD231PS Y13, Y14, Y7 + ADDQ $4, SI + ADDQ $64, DI + DECQ CX + JNZ uakloop + uastore: + VADDPS (DX), Y0, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y1, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R9, DX + VADDPS (DX), Y2, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y3, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R9, DX + VADDPS (DX), Y4, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y5, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R9, DX + VADDPS (DX), Y6, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y7, Y15 + VMOVUPS Y15, 32(DX) + VZEROUPPER + RET + +TEXT ·gemmUA4x8FMA(SB), NOSPLIT, $0-48 + MOVQ kc+0(FP), CX + MOVQ a+8(FP), SI + MOVQ lda+16(FP), R8 + MOVQ pb+24(FP), DI + MOVQ c+32(FP), DX + MOVQ ldc+40(FP), R9 + SHLQ $3, R8 + SHLQ $3, R9 + LEAQ (R8)(R8*2), R11 + LEAQ (R8)(R8*4), R12 + MOVQ DX, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + VXORPD Y0, Y0, Y0 + VXORPD Y1, Y1, Y1 + VXORPD Y2, Y2, Y2 + VXORPD Y3, Y3, Y3 + VXORPD Y4, Y4, Y4 + VXORPD Y5, Y5, Y5 + VXORPD Y6, Y6, Y6 + VXORPD Y7, Y7, Y7 + TESTQ CX, CX + JZ uastore + uakloop: + VMOVUPD (DI), Y12 + VMOVUPD 32(DI), Y13 + VBROADCASTSD (SI), Y14 + VFMADD231PD Y12, Y14, Y0 + VFMADD231PD Y13, Y14, Y1 + VBROADCASTSD (SI)(R8*1), Y14 + VFMADD231PD Y12, Y14, Y2 + VFMADD231PD Y13, Y14, Y3 + VBROADCASTSD (SI)(R8*2), Y14 + VFMADD231PD Y12, Y14, Y4 + VFMADD231PD Y13, Y14, Y5 + VBROADCASTSD (SI)(R11*1), Y14 + VFMADD231PD Y12, Y14, Y6 + VFMADD231PD Y13, Y14, Y7 + ADDQ $8, SI + ADDQ $64, DI + DECQ CX + JNZ uakloop + uastore: + VADDPD (DX), Y0, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y1, Y15 + VMOVUPD Y15, 32(DX) + ADDQ R9, DX + VADDPD (DX), Y2, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y3, Y15 + VMOVUPD Y15, 32(DX) + ADDQ R9, DX + VADDPD (DX), Y4, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y5, Y15 + VMOVUPD Y15, 32(DX) + ADDQ R9, DX + VADDPD (DX), Y6, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y7, Y15 + VMOVUPD Y15, 32(DX) + VZEROUPPER + RET + +TEXT ·gemmUA5x16FMA(SB), NOSPLIT, $0-48 + MOVQ kc+0(FP), CX + MOVQ a+8(FP), SI + MOVQ lda+16(FP), R8 + MOVQ pb+24(FP), DI + MOVQ c+32(FP), DX + MOVQ ldc+40(FP), R9 + SHLQ $2, R8 + SHLQ $2, R9 + LEAQ (R8)(R8*2), R11 + LEAQ (R8)(R8*4), R12 + MOVQ DX, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + VXORPS Y0, Y0, Y0 + VXORPS Y1, Y1, Y1 + VXORPS Y2, Y2, Y2 + VXORPS Y3, Y3, Y3 + VXORPS Y4, Y4, Y4 + VXORPS Y5, Y5, Y5 + VXORPS Y6, Y6, Y6 + VXORPS Y7, Y7, Y7 + VXORPS Y8, Y8, Y8 + VXORPS Y9, Y9, Y9 + TESTQ CX, CX + JZ uastore + uakloop: + VMOVUPS (DI), Y12 + VMOVUPS 32(DI), Y13 + VBROADCASTSS (SI), Y14 + VFMADD231PS Y12, Y14, Y0 + VFMADD231PS Y13, Y14, Y1 + VBROADCASTSS (SI)(R8*1), Y14 + VFMADD231PS Y12, Y14, Y2 + VFMADD231PS Y13, Y14, Y3 + VBROADCASTSS (SI)(R8*2), Y14 + VFMADD231PS Y12, Y14, Y4 + VFMADD231PS Y13, Y14, Y5 + VBROADCASTSS (SI)(R11*1), Y14 + VFMADD231PS Y12, Y14, Y6 + VFMADD231PS Y13, Y14, Y7 + VBROADCASTSS (SI)(R8*4), Y14 + VFMADD231PS Y12, Y14, Y8 + VFMADD231PS Y13, Y14, Y9 + ADDQ $4, SI + ADDQ $64, DI + DECQ CX + JNZ uakloop + uastore: + VADDPS (DX), Y0, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y1, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R9, DX + VADDPS (DX), Y2, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y3, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R9, DX + VADDPS (DX), Y4, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y5, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R9, DX + VADDPS (DX), Y6, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y7, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R9, DX + VADDPS (DX), Y8, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y9, Y15 + VMOVUPS Y15, 32(DX) + VZEROUPPER + RET + +TEXT ·gemmUA5x8FMA(SB), NOSPLIT, $0-48 + MOVQ kc+0(FP), CX + MOVQ a+8(FP), SI + MOVQ lda+16(FP), R8 + MOVQ pb+24(FP), DI + MOVQ c+32(FP), DX + MOVQ ldc+40(FP), R9 + SHLQ $3, R8 + SHLQ $3, R9 + LEAQ (R8)(R8*2), R11 + LEAQ (R8)(R8*4), R12 + MOVQ DX, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + VXORPD Y0, Y0, Y0 + VXORPD Y1, Y1, Y1 + VXORPD Y2, Y2, Y2 + VXORPD Y3, Y3, Y3 + VXORPD Y4, Y4, Y4 + VXORPD Y5, Y5, Y5 + VXORPD Y6, Y6, Y6 + VXORPD Y7, Y7, Y7 + VXORPD Y8, Y8, Y8 + VXORPD Y9, Y9, Y9 + TESTQ CX, CX + JZ uastore + uakloop: + VMOVUPD (DI), Y12 + VMOVUPD 32(DI), Y13 + VBROADCASTSD (SI), Y14 + VFMADD231PD Y12, Y14, Y0 + VFMADD231PD Y13, Y14, Y1 + VBROADCASTSD (SI)(R8*1), Y14 + VFMADD231PD Y12, Y14, Y2 + VFMADD231PD Y13, Y14, Y3 + VBROADCASTSD (SI)(R8*2), Y14 + VFMADD231PD Y12, Y14, Y4 + VFMADD231PD Y13, Y14, Y5 + VBROADCASTSD (SI)(R11*1), Y14 + VFMADD231PD Y12, Y14, Y6 + VFMADD231PD Y13, Y14, Y7 + VBROADCASTSD (SI)(R8*4), Y14 + VFMADD231PD Y12, Y14, Y8 + VFMADD231PD Y13, Y14, Y9 + ADDQ $8, SI + ADDQ $64, DI + DECQ CX + JNZ uakloop + uastore: + VADDPD (DX), Y0, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y1, Y15 + VMOVUPD Y15, 32(DX) + ADDQ R9, DX + VADDPD (DX), Y2, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y3, Y15 + VMOVUPD Y15, 32(DX) + ADDQ R9, DX + VADDPD (DX), Y4, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y5, Y15 + VMOVUPD Y15, 32(DX) + ADDQ R9, DX + VADDPD (DX), Y6, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y7, Y15 + VMOVUPD Y15, 32(DX) + ADDQ R9, DX + VADDPD (DX), Y8, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y9, Y15 + VMOVUPD Y15, 32(DX) + VZEROUPPER + RET + +TEXT ·gemmUA6x16FMA(SB), NOSPLIT, $0-48 + MOVQ kc+0(FP), CX + MOVQ a+8(FP), SI + MOVQ lda+16(FP), R8 + MOVQ pb+24(FP), DI + MOVQ c+32(FP), DX + MOVQ ldc+40(FP), R9 + SHLQ $2, R8 + SHLQ $2, R9 + LEAQ (R8)(R8*2), R11 + LEAQ (R8)(R8*4), R12 + MOVQ DX, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + VXORPS Y0, Y0, Y0 + VXORPS Y1, Y1, Y1 + VXORPS Y2, Y2, Y2 + VXORPS Y3, Y3, Y3 + VXORPS Y4, Y4, Y4 + VXORPS Y5, Y5, Y5 + VXORPS Y6, Y6, Y6 + VXORPS Y7, Y7, Y7 + VXORPS Y8, Y8, Y8 + VXORPS Y9, Y9, Y9 + VXORPS Y10, Y10, Y10 + VXORPS Y11, Y11, Y11 + TESTQ CX, CX + JZ uastore + uakloop: + VMOVUPS (DI), Y12 + VMOVUPS 32(DI), Y13 + VBROADCASTSS (SI), Y14 + VFMADD231PS Y12, Y14, Y0 + VFMADD231PS Y13, Y14, Y1 + VBROADCASTSS (SI)(R8*1), Y14 + VFMADD231PS Y12, Y14, Y2 + VFMADD231PS Y13, Y14, Y3 + VBROADCASTSS (SI)(R8*2), Y14 + VFMADD231PS Y12, Y14, Y4 + VFMADD231PS Y13, Y14, Y5 + VBROADCASTSS (SI)(R11*1), Y14 + VFMADD231PS Y12, Y14, Y6 + VFMADD231PS Y13, Y14, Y7 + VBROADCASTSS (SI)(R8*4), Y14 + VFMADD231PS Y12, Y14, Y8 + VFMADD231PS Y13, Y14, Y9 + VBROADCASTSS (SI)(R12*1), Y14 + VFMADD231PS Y12, Y14, Y10 + VFMADD231PS Y13, Y14, Y11 + ADDQ $4, SI + ADDQ $64, DI + DECQ CX + JNZ uakloop + uastore: + VADDPS (DX), Y0, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y1, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R9, DX + VADDPS (DX), Y2, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y3, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R9, DX + VADDPS (DX), Y4, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y5, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R9, DX + VADDPS (DX), Y6, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y7, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R9, DX + VADDPS (DX), Y8, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y9, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R9, DX + VADDPS (DX), Y10, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y11, Y15 + VMOVUPS Y15, 32(DX) + VZEROUPPER + RET + +TEXT ·gemmUA6x8FMA(SB), NOSPLIT, $0-48 + MOVQ kc+0(FP), CX + MOVQ a+8(FP), SI + MOVQ lda+16(FP), R8 + MOVQ pb+24(FP), DI + MOVQ c+32(FP), DX + MOVQ ldc+40(FP), R9 + SHLQ $3, R8 + SHLQ $3, R9 + LEAQ (R8)(R8*2), R11 + LEAQ (R8)(R8*4), R12 + MOVQ DX, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + ADDQ R9, R10 + PREFETCHT0 (R10) + PREFETCHT0 63(R10) + VXORPD Y0, Y0, Y0 + VXORPD Y1, Y1, Y1 + VXORPD Y2, Y2, Y2 + VXORPD Y3, Y3, Y3 + VXORPD Y4, Y4, Y4 + VXORPD Y5, Y5, Y5 + VXORPD Y6, Y6, Y6 + VXORPD Y7, Y7, Y7 + VXORPD Y8, Y8, Y8 + VXORPD Y9, Y9, Y9 + VXORPD Y10, Y10, Y10 + VXORPD Y11, Y11, Y11 + TESTQ CX, CX + JZ uastore + uakloop: + VMOVUPD (DI), Y12 + VMOVUPD 32(DI), Y13 + VBROADCASTSD (SI), Y14 + VFMADD231PD Y12, Y14, Y0 + VFMADD231PD Y13, Y14, Y1 + VBROADCASTSD (SI)(R8*1), Y14 + VFMADD231PD Y12, Y14, Y2 + VFMADD231PD Y13, Y14, Y3 + VBROADCASTSD (SI)(R8*2), Y14 + VFMADD231PD Y12, Y14, Y4 + VFMADD231PD Y13, Y14, Y5 + VBROADCASTSD (SI)(R11*1), Y14 + VFMADD231PD Y12, Y14, Y6 + VFMADD231PD Y13, Y14, Y7 + VBROADCASTSD (SI)(R8*4), Y14 + VFMADD231PD Y12, Y14, Y8 + VFMADD231PD Y13, Y14, Y9 + VBROADCASTSD (SI)(R12*1), Y14 + VFMADD231PD Y12, Y14, Y10 + VFMADD231PD Y13, Y14, Y11 + ADDQ $8, SI + ADDQ $64, DI + DECQ CX + JNZ uakloop + uastore: + VADDPD (DX), Y0, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y1, Y15 + VMOVUPD Y15, 32(DX) + ADDQ R9, DX + VADDPD (DX), Y2, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y3, Y15 + VMOVUPD Y15, 32(DX) + ADDQ R9, DX + VADDPD (DX), Y4, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y5, Y15 + VMOVUPD Y15, 32(DX) + ADDQ R9, DX + VADDPD (DX), Y6, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y7, Y15 + VMOVUPD Y15, 32(DX) + ADDQ R9, DX + VADDPD (DX), Y8, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y9, Y15 + VMOVUPD Y15, 32(DX) + ADDQ R9, DX + VADDPD (DX), Y10, Y15 + VMOVUPD Y15, (DX) + VADDPD 32(DX), Y11, Y15 + VMOVUPD Y15, 32(DX) + VZEROUPPER + RET + TEXT ·dotFMA(SB), NOSPLIT, $0-32 MOVQ a+0(FP), SI MOVQ b+8(FP), DI diff --git a/internal/kernels/sum_arm64.s b/internal/kernels/sum_arm64.s index b938caf..8588dc0 100644 --- a/internal/kernels/sum_arm64.s +++ b/internal/kernels/sum_arm64.s @@ -687,6 +687,1302 @@ TEXT ·gemmMicro8x8S(SB), NOSPLIT, $0-40 VST1 [V16.S4, V17.S4], (R3) RET +TEXT ·gemmUA1x8S(SB), NOSPLIT, $0-48 + MOVD kc+0(FP), R0 + MOVD a+8(FP), R1 + MOVD lda+16(FP), R2 + MOVD pb+24(FP), R13 + MOVD c+32(FP), R3 + MOVD ldc+40(FP), R4 + LSL $2, R2, R2 + LSL $2, R4, R4 + MOVD R1, R5 + VEOR V0.B16, V0.B16, V0.B16 + VEOR V1.B16, V1.B16, V1.B16 + LSR $2, R0, R14 + AND $3, R0, R15 + CBZ R14, uatail + uakloop: + VLD1.P 16(R5), [V20.S4] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] + WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] + WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] + WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] + WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] + SUB $1, R14 + CBNZ R14, uakloop + uatail: + CBZ R15, uastore + uatloop: + FMOVS.P 4(R5), F20 + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] + WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] + SUB $1, R15 + CBNZ R15, uatloop + uastore: + VLD1 (R3), [V16.S4, V17.S4] + VFADD V0.S4, V16.S4, V16.S4 + VFADD V1.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + RET + +TEXT ·gemmUA1x8D(SB), NOSPLIT, $0-48 + MOVD kc+0(FP), R0 + MOVD a+8(FP), R1 + MOVD lda+16(FP), R2 + MOVD pb+24(FP), R13 + MOVD c+32(FP), R3 + MOVD ldc+40(FP), R4 + LSL $3, R2, R2 + LSL $3, R4, R4 + MOVD R1, R5 + VEOR V0.B16, V0.B16, V0.B16 + VEOR V1.B16, V1.B16, V1.B16 + VEOR V2.B16, V2.B16, V2.B16 + VEOR V3.B16, V3.B16, V3.B16 + LSR $1, R0, R14 + AND $1, R0, R15 + CBZ R14, uatail + uakloop: + VLD1.P 16(R5), [V20.D2] + VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] + WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] + WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] + WORD $0x4FD41263 // FMLA V3.2D, V19.2D, V20.D[0] + VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + WORD $0x4FD41A00 // FMLA V0.2D, V16.2D, V20.D[1] + WORD $0x4FD41A21 // FMLA V1.2D, V17.2D, V20.D[1] + WORD $0x4FD41A42 // FMLA V2.2D, V18.2D, V20.D[1] + WORD $0x4FD41A63 // FMLA V3.2D, V19.2D, V20.D[1] + SUB $1, R14 + CBNZ R14, uakloop + uatail: + CBZ R15, uastore + uatloop: + FMOVD.P 8(R5), F20 + VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] + WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] + WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] + WORD $0x4FD41263 // FMLA V3.2D, V19.2D, V20.D[0] + SUB $1, R15 + CBNZ R15, uatloop + uastore: + VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] + VFADD V0.D2, V16.D2, V16.D2 + VFADD V1.D2, V17.D2, V17.D2 + VFADD V2.D2, V18.D2, V18.D2 + VFADD V3.D2, V19.D2, V19.D2 + VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) + RET + +TEXT ·gemmUA2x8S(SB), NOSPLIT, $0-48 + MOVD kc+0(FP), R0 + MOVD a+8(FP), R1 + MOVD lda+16(FP), R2 + MOVD pb+24(FP), R13 + MOVD c+32(FP), R3 + MOVD ldc+40(FP), R4 + LSL $2, R2, R2 + LSL $2, R4, R4 + MOVD R1, R5 + ADD R2, R5, R6 + VEOR V0.B16, V0.B16, V0.B16 + VEOR V1.B16, V1.B16, V1.B16 + VEOR V2.B16, V2.B16, V2.B16 + VEOR V3.B16, V3.B16, V3.B16 + LSR $2, R0, R14 + AND $3, R0, R15 + CBZ R14, uatail + uakloop: + VLD1.P 16(R5), [V20.S4] + VLD1.P 16(R6), [V21.S4] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] + WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] + WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] + WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] + WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] + WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] + WORD $0x4FB51223 // FMLA V3.4S, V17.4S, V21.S[1] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] + WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] + WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] + WORD $0x4F951A23 // FMLA V3.4S, V17.4S, V21.S[2] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] + WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] + WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] + WORD $0x4FB51A23 // FMLA V3.4S, V17.4S, V21.S[3] + SUB $1, R14 + CBNZ R14, uakloop + uatail: + CBZ R15, uastore + uatloop: + FMOVS.P 4(R5), F20 + FMOVS.P 4(R6), F21 + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] + WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] + WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] + WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] + SUB $1, R15 + CBNZ R15, uatloop + uastore: + VLD1 (R3), [V16.S4, V17.S4] + VFADD V0.S4, V16.S4, V16.S4 + VFADD V1.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V2.S4, V16.S4, V16.S4 + VFADD V3.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + RET + +TEXT ·gemmUA2x8D(SB), NOSPLIT, $0-48 + MOVD kc+0(FP), R0 + MOVD a+8(FP), R1 + MOVD lda+16(FP), R2 + MOVD pb+24(FP), R13 + MOVD c+32(FP), R3 + MOVD ldc+40(FP), R4 + LSL $3, R2, R2 + LSL $3, R4, R4 + MOVD R1, R5 + ADD R2, R5, R6 + VEOR V0.B16, V0.B16, V0.B16 + VEOR V1.B16, V1.B16, V1.B16 + VEOR V2.B16, V2.B16, V2.B16 + VEOR V3.B16, V3.B16, V3.B16 + VEOR V4.B16, V4.B16, V4.B16 + VEOR V5.B16, V5.B16, V5.B16 + VEOR V6.B16, V6.B16, V6.B16 + VEOR V7.B16, V7.B16, V7.B16 + LSR $1, R0, R14 + AND $1, R0, R15 + CBZ R14, uatail + uakloop: + VLD1.P 16(R5), [V20.D2] + VLD1.P 16(R6), [V21.D2] + VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] + WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] + WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] + WORD $0x4FD41263 // FMLA V3.2D, V19.2D, V20.D[0] + WORD $0x4FD51204 // FMLA V4.2D, V16.2D, V21.D[0] + WORD $0x4FD51225 // FMLA V5.2D, V17.2D, V21.D[0] + WORD $0x4FD51246 // FMLA V6.2D, V18.2D, V21.D[0] + WORD $0x4FD51267 // FMLA V7.2D, V19.2D, V21.D[0] + VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + WORD $0x4FD41A00 // FMLA V0.2D, V16.2D, V20.D[1] + WORD $0x4FD41A21 // FMLA V1.2D, V17.2D, V20.D[1] + WORD $0x4FD41A42 // FMLA V2.2D, V18.2D, V20.D[1] + WORD $0x4FD41A63 // FMLA V3.2D, V19.2D, V20.D[1] + WORD $0x4FD51A04 // FMLA V4.2D, V16.2D, V21.D[1] + WORD $0x4FD51A25 // FMLA V5.2D, V17.2D, V21.D[1] + WORD $0x4FD51A46 // FMLA V6.2D, V18.2D, V21.D[1] + WORD $0x4FD51A67 // FMLA V7.2D, V19.2D, V21.D[1] + SUB $1, R14 + CBNZ R14, uakloop + uatail: + CBZ R15, uastore + uatloop: + FMOVD.P 8(R5), F20 + FMOVD.P 8(R6), F21 + VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] + WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] + WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] + WORD $0x4FD41263 // FMLA V3.2D, V19.2D, V20.D[0] + WORD $0x4FD51204 // FMLA V4.2D, V16.2D, V21.D[0] + WORD $0x4FD51225 // FMLA V5.2D, V17.2D, V21.D[0] + WORD $0x4FD51246 // FMLA V6.2D, V18.2D, V21.D[0] + WORD $0x4FD51267 // FMLA V7.2D, V19.2D, V21.D[0] + SUB $1, R15 + CBNZ R15, uatloop + uastore: + VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] + VFADD V0.D2, V16.D2, V16.D2 + VFADD V1.D2, V17.D2, V17.D2 + VFADD V2.D2, V18.D2, V18.D2 + VFADD V3.D2, V19.D2, V19.D2 + VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] + VFADD V4.D2, V16.D2, V16.D2 + VFADD V5.D2, V17.D2, V17.D2 + VFADD V6.D2, V18.D2, V18.D2 + VFADD V7.D2, V19.D2, V19.D2 + VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) + RET + +TEXT ·gemmUA3x8S(SB), NOSPLIT, $0-48 + MOVD kc+0(FP), R0 + MOVD a+8(FP), R1 + MOVD lda+16(FP), R2 + MOVD pb+24(FP), R13 + MOVD c+32(FP), R3 + MOVD ldc+40(FP), R4 + LSL $2, R2, R2 + LSL $2, R4, R4 + MOVD R1, R5 + ADD R2, R5, R6 + ADD R2, R6, R7 + VEOR V0.B16, V0.B16, V0.B16 + VEOR V1.B16, V1.B16, V1.B16 + VEOR V2.B16, V2.B16, V2.B16 + VEOR V3.B16, V3.B16, V3.B16 + VEOR V4.B16, V4.B16, V4.B16 + VEOR V5.B16, V5.B16, V5.B16 + LSR $2, R0, R14 + AND $3, R0, R15 + CBZ R14, uatail + uakloop: + VLD1.P 16(R5), [V20.S4] + VLD1.P 16(R6), [V21.S4] + VLD1.P 16(R7), [V22.S4] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] + WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] + WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] + WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] + WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] + WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] + WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] + WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] + WORD $0x4FB51223 // FMLA V3.4S, V17.4S, V21.S[1] + WORD $0x4FB61204 // FMLA V4.4S, V16.4S, V22.S[1] + WORD $0x4FB61225 // FMLA V5.4S, V17.4S, V22.S[1] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] + WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] + WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] + WORD $0x4F951A23 // FMLA V3.4S, V17.4S, V21.S[2] + WORD $0x4F961A04 // FMLA V4.4S, V16.4S, V22.S[2] + WORD $0x4F961A25 // FMLA V5.4S, V17.4S, V22.S[2] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] + WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] + WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] + WORD $0x4FB51A23 // FMLA V3.4S, V17.4S, V21.S[3] + WORD $0x4FB61A04 // FMLA V4.4S, V16.4S, V22.S[3] + WORD $0x4FB61A25 // FMLA V5.4S, V17.4S, V22.S[3] + SUB $1, R14 + CBNZ R14, uakloop + uatail: + CBZ R15, uastore + uatloop: + FMOVS.P 4(R5), F20 + FMOVS.P 4(R6), F21 + FMOVS.P 4(R7), F22 + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] + WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] + WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] + WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] + WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] + WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] + SUB $1, R15 + CBNZ R15, uatloop + uastore: + VLD1 (R3), [V16.S4, V17.S4] + VFADD V0.S4, V16.S4, V16.S4 + VFADD V1.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V2.S4, V16.S4, V16.S4 + VFADD V3.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V4.S4, V16.S4, V16.S4 + VFADD V5.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + RET + +TEXT ·gemmUA3x8D(SB), NOSPLIT, $0-48 + MOVD kc+0(FP), R0 + MOVD a+8(FP), R1 + MOVD lda+16(FP), R2 + MOVD pb+24(FP), R13 + MOVD c+32(FP), R3 + MOVD ldc+40(FP), R4 + LSL $3, R2, R2 + LSL $3, R4, R4 + MOVD R1, R5 + ADD R2, R5, R6 + ADD R2, R6, R7 + VEOR V0.B16, V0.B16, V0.B16 + VEOR V1.B16, V1.B16, V1.B16 + VEOR V2.B16, V2.B16, V2.B16 + VEOR V3.B16, V3.B16, V3.B16 + VEOR V4.B16, V4.B16, V4.B16 + VEOR V5.B16, V5.B16, V5.B16 + VEOR V6.B16, V6.B16, V6.B16 + VEOR V7.B16, V7.B16, V7.B16 + VEOR V8.B16, V8.B16, V8.B16 + VEOR V9.B16, V9.B16, V9.B16 + VEOR V10.B16, V10.B16, V10.B16 + VEOR V11.B16, V11.B16, V11.B16 + LSR $1, R0, R14 + AND $1, R0, R15 + CBZ R14, uatail + uakloop: + VLD1.P 16(R5), [V20.D2] + VLD1.P 16(R6), [V21.D2] + VLD1.P 16(R7), [V22.D2] + VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] + WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] + WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] + WORD $0x4FD41263 // FMLA V3.2D, V19.2D, V20.D[0] + WORD $0x4FD51204 // FMLA V4.2D, V16.2D, V21.D[0] + WORD $0x4FD51225 // FMLA V5.2D, V17.2D, V21.D[0] + WORD $0x4FD51246 // FMLA V6.2D, V18.2D, V21.D[0] + WORD $0x4FD51267 // FMLA V7.2D, V19.2D, V21.D[0] + WORD $0x4FD61208 // FMLA V8.2D, V16.2D, V22.D[0] + WORD $0x4FD61229 // FMLA V9.2D, V17.2D, V22.D[0] + WORD $0x4FD6124A // FMLA V10.2D, V18.2D, V22.D[0] + WORD $0x4FD6126B // FMLA V11.2D, V19.2D, V22.D[0] + VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + WORD $0x4FD41A00 // FMLA V0.2D, V16.2D, V20.D[1] + WORD $0x4FD41A21 // FMLA V1.2D, V17.2D, V20.D[1] + WORD $0x4FD41A42 // FMLA V2.2D, V18.2D, V20.D[1] + WORD $0x4FD41A63 // FMLA V3.2D, V19.2D, V20.D[1] + WORD $0x4FD51A04 // FMLA V4.2D, V16.2D, V21.D[1] + WORD $0x4FD51A25 // FMLA V5.2D, V17.2D, V21.D[1] + WORD $0x4FD51A46 // FMLA V6.2D, V18.2D, V21.D[1] + WORD $0x4FD51A67 // FMLA V7.2D, V19.2D, V21.D[1] + WORD $0x4FD61A08 // FMLA V8.2D, V16.2D, V22.D[1] + WORD $0x4FD61A29 // FMLA V9.2D, V17.2D, V22.D[1] + WORD $0x4FD61A4A // FMLA V10.2D, V18.2D, V22.D[1] + WORD $0x4FD61A6B // FMLA V11.2D, V19.2D, V22.D[1] + SUB $1, R14 + CBNZ R14, uakloop + uatail: + CBZ R15, uastore + uatloop: + FMOVD.P 8(R5), F20 + FMOVD.P 8(R6), F21 + FMOVD.P 8(R7), F22 + VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] + WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] + WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] + WORD $0x4FD41263 // FMLA V3.2D, V19.2D, V20.D[0] + WORD $0x4FD51204 // FMLA V4.2D, V16.2D, V21.D[0] + WORD $0x4FD51225 // FMLA V5.2D, V17.2D, V21.D[0] + WORD $0x4FD51246 // FMLA V6.2D, V18.2D, V21.D[0] + WORD $0x4FD51267 // FMLA V7.2D, V19.2D, V21.D[0] + WORD $0x4FD61208 // FMLA V8.2D, V16.2D, V22.D[0] + WORD $0x4FD61229 // FMLA V9.2D, V17.2D, V22.D[0] + WORD $0x4FD6124A // FMLA V10.2D, V18.2D, V22.D[0] + WORD $0x4FD6126B // FMLA V11.2D, V19.2D, V22.D[0] + SUB $1, R15 + CBNZ R15, uatloop + uastore: + VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] + VFADD V0.D2, V16.D2, V16.D2 + VFADD V1.D2, V17.D2, V17.D2 + VFADD V2.D2, V18.D2, V18.D2 + VFADD V3.D2, V19.D2, V19.D2 + VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] + VFADD V4.D2, V16.D2, V16.D2 + VFADD V5.D2, V17.D2, V17.D2 + VFADD V6.D2, V18.D2, V18.D2 + VFADD V7.D2, V19.D2, V19.D2 + VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] + VFADD V8.D2, V16.D2, V16.D2 + VFADD V9.D2, V17.D2, V17.D2 + VFADD V10.D2, V18.D2, V18.D2 + VFADD V11.D2, V19.D2, V19.D2 + VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) + RET + +TEXT ·gemmUA4x8S(SB), NOSPLIT, $0-48 + MOVD kc+0(FP), R0 + MOVD a+8(FP), R1 + MOVD lda+16(FP), R2 + MOVD pb+24(FP), R13 + MOVD c+32(FP), R3 + MOVD ldc+40(FP), R4 + LSL $2, R2, R2 + LSL $2, R4, R4 + MOVD R1, R5 + ADD R2, R5, R6 + ADD R2, R6, R7 + ADD R2, R7, R8 + VEOR V0.B16, V0.B16, V0.B16 + VEOR V1.B16, V1.B16, V1.B16 + VEOR V2.B16, V2.B16, V2.B16 + VEOR V3.B16, V3.B16, V3.B16 + VEOR V4.B16, V4.B16, V4.B16 + VEOR V5.B16, V5.B16, V5.B16 + VEOR V6.B16, V6.B16, V6.B16 + VEOR V7.B16, V7.B16, V7.B16 + LSR $2, R0, R14 + AND $3, R0, R15 + CBZ R14, uatail + uakloop: + VLD1.P 16(R5), [V20.S4] + VLD1.P 16(R6), [V21.S4] + VLD1.P 16(R7), [V22.S4] + VLD1.P 16(R8), [V23.S4] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] + WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] + WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] + WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] + WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] + WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] + WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] + WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] + WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] + WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] + WORD $0x4FB51223 // FMLA V3.4S, V17.4S, V21.S[1] + WORD $0x4FB61204 // FMLA V4.4S, V16.4S, V22.S[1] + WORD $0x4FB61225 // FMLA V5.4S, V17.4S, V22.S[1] + WORD $0x4FB71206 // FMLA V6.4S, V16.4S, V23.S[1] + WORD $0x4FB71227 // FMLA V7.4S, V17.4S, V23.S[1] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] + WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] + WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] + WORD $0x4F951A23 // FMLA V3.4S, V17.4S, V21.S[2] + WORD $0x4F961A04 // FMLA V4.4S, V16.4S, V22.S[2] + WORD $0x4F961A25 // FMLA V5.4S, V17.4S, V22.S[2] + WORD $0x4F971A06 // FMLA V6.4S, V16.4S, V23.S[2] + WORD $0x4F971A27 // FMLA V7.4S, V17.4S, V23.S[2] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] + WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] + WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] + WORD $0x4FB51A23 // FMLA V3.4S, V17.4S, V21.S[3] + WORD $0x4FB61A04 // FMLA V4.4S, V16.4S, V22.S[3] + WORD $0x4FB61A25 // FMLA V5.4S, V17.4S, V22.S[3] + WORD $0x4FB71A06 // FMLA V6.4S, V16.4S, V23.S[3] + WORD $0x4FB71A27 // FMLA V7.4S, V17.4S, V23.S[3] + SUB $1, R14 + CBNZ R14, uakloop + uatail: + CBZ R15, uastore + uatloop: + FMOVS.P 4(R5), F20 + FMOVS.P 4(R6), F21 + FMOVS.P 4(R7), F22 + FMOVS.P 4(R8), F23 + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] + WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] + WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] + WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] + WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] + WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] + WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] + WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] + SUB $1, R15 + CBNZ R15, uatloop + uastore: + VLD1 (R3), [V16.S4, V17.S4] + VFADD V0.S4, V16.S4, V16.S4 + VFADD V1.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V2.S4, V16.S4, V16.S4 + VFADD V3.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V4.S4, V16.S4, V16.S4 + VFADD V5.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V6.S4, V16.S4, V16.S4 + VFADD V7.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + RET + +TEXT ·gemmUA4x8D(SB), NOSPLIT, $0-48 + MOVD kc+0(FP), R0 + MOVD a+8(FP), R1 + MOVD lda+16(FP), R2 + MOVD pb+24(FP), R13 + MOVD c+32(FP), R3 + MOVD ldc+40(FP), R4 + LSL $3, R2, R2 + LSL $3, R4, R4 + MOVD R1, R5 + ADD R2, R5, R6 + ADD R2, R6, R7 + ADD R2, R7, R8 + VEOR V0.B16, V0.B16, V0.B16 + VEOR V1.B16, V1.B16, V1.B16 + VEOR V2.B16, V2.B16, V2.B16 + VEOR V3.B16, V3.B16, V3.B16 + VEOR V4.B16, V4.B16, V4.B16 + VEOR V5.B16, V5.B16, V5.B16 + VEOR V6.B16, V6.B16, V6.B16 + VEOR V7.B16, V7.B16, V7.B16 + VEOR V8.B16, V8.B16, V8.B16 + VEOR V9.B16, V9.B16, V9.B16 + VEOR V10.B16, V10.B16, V10.B16 + VEOR V11.B16, V11.B16, V11.B16 + VEOR V12.B16, V12.B16, V12.B16 + VEOR V13.B16, V13.B16, V13.B16 + VEOR V14.B16, V14.B16, V14.B16 + VEOR V15.B16, V15.B16, V15.B16 + LSR $1, R0, R14 + AND $1, R0, R15 + CBZ R14, uatail + uakloop: + VLD1.P 16(R5), [V20.D2] + VLD1.P 16(R6), [V21.D2] + VLD1.P 16(R7), [V22.D2] + VLD1.P 16(R8), [V23.D2] + VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] + WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] + WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] + WORD $0x4FD41263 // FMLA V3.2D, V19.2D, V20.D[0] + WORD $0x4FD51204 // FMLA V4.2D, V16.2D, V21.D[0] + WORD $0x4FD51225 // FMLA V5.2D, V17.2D, V21.D[0] + WORD $0x4FD51246 // FMLA V6.2D, V18.2D, V21.D[0] + WORD $0x4FD51267 // FMLA V7.2D, V19.2D, V21.D[0] + WORD $0x4FD61208 // FMLA V8.2D, V16.2D, V22.D[0] + WORD $0x4FD61229 // FMLA V9.2D, V17.2D, V22.D[0] + WORD $0x4FD6124A // FMLA V10.2D, V18.2D, V22.D[0] + WORD $0x4FD6126B // FMLA V11.2D, V19.2D, V22.D[0] + WORD $0x4FD7120C // FMLA V12.2D, V16.2D, V23.D[0] + WORD $0x4FD7122D // FMLA V13.2D, V17.2D, V23.D[0] + WORD $0x4FD7124E // FMLA V14.2D, V18.2D, V23.D[0] + WORD $0x4FD7126F // FMLA V15.2D, V19.2D, V23.D[0] + VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + WORD $0x4FD41A00 // FMLA V0.2D, V16.2D, V20.D[1] + WORD $0x4FD41A21 // FMLA V1.2D, V17.2D, V20.D[1] + WORD $0x4FD41A42 // FMLA V2.2D, V18.2D, V20.D[1] + WORD $0x4FD41A63 // FMLA V3.2D, V19.2D, V20.D[1] + WORD $0x4FD51A04 // FMLA V4.2D, V16.2D, V21.D[1] + WORD $0x4FD51A25 // FMLA V5.2D, V17.2D, V21.D[1] + WORD $0x4FD51A46 // FMLA V6.2D, V18.2D, V21.D[1] + WORD $0x4FD51A67 // FMLA V7.2D, V19.2D, V21.D[1] + WORD $0x4FD61A08 // FMLA V8.2D, V16.2D, V22.D[1] + WORD $0x4FD61A29 // FMLA V9.2D, V17.2D, V22.D[1] + WORD $0x4FD61A4A // FMLA V10.2D, V18.2D, V22.D[1] + WORD $0x4FD61A6B // FMLA V11.2D, V19.2D, V22.D[1] + WORD $0x4FD71A0C // FMLA V12.2D, V16.2D, V23.D[1] + WORD $0x4FD71A2D // FMLA V13.2D, V17.2D, V23.D[1] + WORD $0x4FD71A4E // FMLA V14.2D, V18.2D, V23.D[1] + WORD $0x4FD71A6F // FMLA V15.2D, V19.2D, V23.D[1] + SUB $1, R14 + CBNZ R14, uakloop + uatail: + CBZ R15, uastore + uatloop: + FMOVD.P 8(R5), F20 + FMOVD.P 8(R6), F21 + FMOVD.P 8(R7), F22 + FMOVD.P 8(R8), F23 + VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] + WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] + WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] + WORD $0x4FD41263 // FMLA V3.2D, V19.2D, V20.D[0] + WORD $0x4FD51204 // FMLA V4.2D, V16.2D, V21.D[0] + WORD $0x4FD51225 // FMLA V5.2D, V17.2D, V21.D[0] + WORD $0x4FD51246 // FMLA V6.2D, V18.2D, V21.D[0] + WORD $0x4FD51267 // FMLA V7.2D, V19.2D, V21.D[0] + WORD $0x4FD61208 // FMLA V8.2D, V16.2D, V22.D[0] + WORD $0x4FD61229 // FMLA V9.2D, V17.2D, V22.D[0] + WORD $0x4FD6124A // FMLA V10.2D, V18.2D, V22.D[0] + WORD $0x4FD6126B // FMLA V11.2D, V19.2D, V22.D[0] + WORD $0x4FD7120C // FMLA V12.2D, V16.2D, V23.D[0] + WORD $0x4FD7122D // FMLA V13.2D, V17.2D, V23.D[0] + WORD $0x4FD7124E // FMLA V14.2D, V18.2D, V23.D[0] + WORD $0x4FD7126F // FMLA V15.2D, V19.2D, V23.D[0] + SUB $1, R15 + CBNZ R15, uatloop + uastore: + VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] + VFADD V0.D2, V16.D2, V16.D2 + VFADD V1.D2, V17.D2, V17.D2 + VFADD V2.D2, V18.D2, V18.D2 + VFADD V3.D2, V19.D2, V19.D2 + VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] + VFADD V4.D2, V16.D2, V16.D2 + VFADD V5.D2, V17.D2, V17.D2 + VFADD V6.D2, V18.D2, V18.D2 + VFADD V7.D2, V19.D2, V19.D2 + VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] + VFADD V8.D2, V16.D2, V16.D2 + VFADD V9.D2, V17.D2, V17.D2 + VFADD V10.D2, V18.D2, V18.D2 + VFADD V11.D2, V19.D2, V19.D2 + VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] + VFADD V12.D2, V16.D2, V16.D2 + VFADD V13.D2, V17.D2, V17.D2 + VFADD V14.D2, V18.D2, V18.D2 + VFADD V15.D2, V19.D2, V19.D2 + VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) + RET + +TEXT ·gemmUA5x8S(SB), NOSPLIT, $0-48 + MOVD kc+0(FP), R0 + MOVD a+8(FP), R1 + MOVD lda+16(FP), R2 + MOVD pb+24(FP), R13 + MOVD c+32(FP), R3 + MOVD ldc+40(FP), R4 + LSL $2, R2, R2 + LSL $2, R4, R4 + MOVD R1, R5 + ADD R2, R5, R6 + ADD R2, R6, R7 + ADD R2, R7, R8 + ADD R2, R8, R9 + VEOR V0.B16, V0.B16, V0.B16 + VEOR V1.B16, V1.B16, V1.B16 + VEOR V2.B16, V2.B16, V2.B16 + VEOR V3.B16, V3.B16, V3.B16 + VEOR V4.B16, V4.B16, V4.B16 + VEOR V5.B16, V5.B16, V5.B16 + VEOR V6.B16, V6.B16, V6.B16 + VEOR V7.B16, V7.B16, V7.B16 + VEOR V8.B16, V8.B16, V8.B16 + VEOR V9.B16, V9.B16, V9.B16 + LSR $2, R0, R14 + AND $3, R0, R15 + CBZ R14, uatail + uakloop: + VLD1.P 16(R5), [V20.S4] + VLD1.P 16(R6), [V21.S4] + VLD1.P 16(R7), [V22.S4] + VLD1.P 16(R8), [V23.S4] + VLD1.P 16(R9), [V24.S4] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] + WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] + WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] + WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] + WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] + WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] + WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] + WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] + WORD $0x4F981208 // FMLA V8.4S, V16.4S, V24.S[0] + WORD $0x4F981229 // FMLA V9.4S, V17.4S, V24.S[0] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] + WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] + WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] + WORD $0x4FB51223 // FMLA V3.4S, V17.4S, V21.S[1] + WORD $0x4FB61204 // FMLA V4.4S, V16.4S, V22.S[1] + WORD $0x4FB61225 // FMLA V5.4S, V17.4S, V22.S[1] + WORD $0x4FB71206 // FMLA V6.4S, V16.4S, V23.S[1] + WORD $0x4FB71227 // FMLA V7.4S, V17.4S, V23.S[1] + WORD $0x4FB81208 // FMLA V8.4S, V16.4S, V24.S[1] + WORD $0x4FB81229 // FMLA V9.4S, V17.4S, V24.S[1] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] + WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] + WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] + WORD $0x4F951A23 // FMLA V3.4S, V17.4S, V21.S[2] + WORD $0x4F961A04 // FMLA V4.4S, V16.4S, V22.S[2] + WORD $0x4F961A25 // FMLA V5.4S, V17.4S, V22.S[2] + WORD $0x4F971A06 // FMLA V6.4S, V16.4S, V23.S[2] + WORD $0x4F971A27 // FMLA V7.4S, V17.4S, V23.S[2] + WORD $0x4F981A08 // FMLA V8.4S, V16.4S, V24.S[2] + WORD $0x4F981A29 // FMLA V9.4S, V17.4S, V24.S[2] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] + WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] + WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] + WORD $0x4FB51A23 // FMLA V3.4S, V17.4S, V21.S[3] + WORD $0x4FB61A04 // FMLA V4.4S, V16.4S, V22.S[3] + WORD $0x4FB61A25 // FMLA V5.4S, V17.4S, V22.S[3] + WORD $0x4FB71A06 // FMLA V6.4S, V16.4S, V23.S[3] + WORD $0x4FB71A27 // FMLA V7.4S, V17.4S, V23.S[3] + WORD $0x4FB81A08 // FMLA V8.4S, V16.4S, V24.S[3] + WORD $0x4FB81A29 // FMLA V9.4S, V17.4S, V24.S[3] + SUB $1, R14 + CBNZ R14, uakloop + uatail: + CBZ R15, uastore + uatloop: + FMOVS.P 4(R5), F20 + FMOVS.P 4(R6), F21 + FMOVS.P 4(R7), F22 + FMOVS.P 4(R8), F23 + FMOVS.P 4(R9), F24 + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] + WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] + WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] + WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] + WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] + WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] + WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] + WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] + WORD $0x4F981208 // FMLA V8.4S, V16.4S, V24.S[0] + WORD $0x4F981229 // FMLA V9.4S, V17.4S, V24.S[0] + SUB $1, R15 + CBNZ R15, uatloop + uastore: + VLD1 (R3), [V16.S4, V17.S4] + VFADD V0.S4, V16.S4, V16.S4 + VFADD V1.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V2.S4, V16.S4, V16.S4 + VFADD V3.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V4.S4, V16.S4, V16.S4 + VFADD V5.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V6.S4, V16.S4, V16.S4 + VFADD V7.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V8.S4, V16.S4, V16.S4 + VFADD V9.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + RET + +TEXT ·gemmUA6x8S(SB), NOSPLIT, $0-48 + MOVD kc+0(FP), R0 + MOVD a+8(FP), R1 + MOVD lda+16(FP), R2 + MOVD pb+24(FP), R13 + MOVD c+32(FP), R3 + MOVD ldc+40(FP), R4 + LSL $2, R2, R2 + LSL $2, R4, R4 + MOVD R1, R5 + ADD R2, R5, R6 + ADD R2, R6, R7 + ADD R2, R7, R8 + ADD R2, R8, R9 + ADD R2, R9, R10 + VEOR V0.B16, V0.B16, V0.B16 + VEOR V1.B16, V1.B16, V1.B16 + VEOR V2.B16, V2.B16, V2.B16 + VEOR V3.B16, V3.B16, V3.B16 + VEOR V4.B16, V4.B16, V4.B16 + VEOR V5.B16, V5.B16, V5.B16 + VEOR V6.B16, V6.B16, V6.B16 + VEOR V7.B16, V7.B16, V7.B16 + VEOR V8.B16, V8.B16, V8.B16 + VEOR V9.B16, V9.B16, V9.B16 + VEOR V10.B16, V10.B16, V10.B16 + VEOR V11.B16, V11.B16, V11.B16 + LSR $2, R0, R14 + AND $3, R0, R15 + CBZ R14, uatail + uakloop: + VLD1.P 16(R5), [V20.S4] + VLD1.P 16(R6), [V21.S4] + VLD1.P 16(R7), [V22.S4] + VLD1.P 16(R8), [V23.S4] + VLD1.P 16(R9), [V24.S4] + VLD1.P 16(R10), [V25.S4] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] + WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] + WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] + WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] + WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] + WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] + WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] + WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] + WORD $0x4F981208 // FMLA V8.4S, V16.4S, V24.S[0] + WORD $0x4F981229 // FMLA V9.4S, V17.4S, V24.S[0] + WORD $0x4F99120A // FMLA V10.4S, V16.4S, V25.S[0] + WORD $0x4F99122B // FMLA V11.4S, V17.4S, V25.S[0] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] + WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] + WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] + WORD $0x4FB51223 // FMLA V3.4S, V17.4S, V21.S[1] + WORD $0x4FB61204 // FMLA V4.4S, V16.4S, V22.S[1] + WORD $0x4FB61225 // FMLA V5.4S, V17.4S, V22.S[1] + WORD $0x4FB71206 // FMLA V6.4S, V16.4S, V23.S[1] + WORD $0x4FB71227 // FMLA V7.4S, V17.4S, V23.S[1] + WORD $0x4FB81208 // FMLA V8.4S, V16.4S, V24.S[1] + WORD $0x4FB81229 // FMLA V9.4S, V17.4S, V24.S[1] + WORD $0x4FB9120A // FMLA V10.4S, V16.4S, V25.S[1] + WORD $0x4FB9122B // FMLA V11.4S, V17.4S, V25.S[1] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] + WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] + WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] + WORD $0x4F951A23 // FMLA V3.4S, V17.4S, V21.S[2] + WORD $0x4F961A04 // FMLA V4.4S, V16.4S, V22.S[2] + WORD $0x4F961A25 // FMLA V5.4S, V17.4S, V22.S[2] + WORD $0x4F971A06 // FMLA V6.4S, V16.4S, V23.S[2] + WORD $0x4F971A27 // FMLA V7.4S, V17.4S, V23.S[2] + WORD $0x4F981A08 // FMLA V8.4S, V16.4S, V24.S[2] + WORD $0x4F981A29 // FMLA V9.4S, V17.4S, V24.S[2] + WORD $0x4F991A0A // FMLA V10.4S, V16.4S, V25.S[2] + WORD $0x4F991A2B // FMLA V11.4S, V17.4S, V25.S[2] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] + WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] + WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] + WORD $0x4FB51A23 // FMLA V3.4S, V17.4S, V21.S[3] + WORD $0x4FB61A04 // FMLA V4.4S, V16.4S, V22.S[3] + WORD $0x4FB61A25 // FMLA V5.4S, V17.4S, V22.S[3] + WORD $0x4FB71A06 // FMLA V6.4S, V16.4S, V23.S[3] + WORD $0x4FB71A27 // FMLA V7.4S, V17.4S, V23.S[3] + WORD $0x4FB81A08 // FMLA V8.4S, V16.4S, V24.S[3] + WORD $0x4FB81A29 // FMLA V9.4S, V17.4S, V24.S[3] + WORD $0x4FB91A0A // FMLA V10.4S, V16.4S, V25.S[3] + WORD $0x4FB91A2B // FMLA V11.4S, V17.4S, V25.S[3] + SUB $1, R14 + CBNZ R14, uakloop + uatail: + CBZ R15, uastore + uatloop: + FMOVS.P 4(R5), F20 + FMOVS.P 4(R6), F21 + FMOVS.P 4(R7), F22 + FMOVS.P 4(R8), F23 + FMOVS.P 4(R9), F24 + FMOVS.P 4(R10), F25 + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] + WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] + WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] + WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] + WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] + WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] + WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] + WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] + WORD $0x4F981208 // FMLA V8.4S, V16.4S, V24.S[0] + WORD $0x4F981229 // FMLA V9.4S, V17.4S, V24.S[0] + WORD $0x4F99120A // FMLA V10.4S, V16.4S, V25.S[0] + WORD $0x4F99122B // FMLA V11.4S, V17.4S, V25.S[0] + SUB $1, R15 + CBNZ R15, uatloop + uastore: + VLD1 (R3), [V16.S4, V17.S4] + VFADD V0.S4, V16.S4, V16.S4 + VFADD V1.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V2.S4, V16.S4, V16.S4 + VFADD V3.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V4.S4, V16.S4, V16.S4 + VFADD V5.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V6.S4, V16.S4, V16.S4 + VFADD V7.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V8.S4, V16.S4, V16.S4 + VFADD V9.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V10.S4, V16.S4, V16.S4 + VFADD V11.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + RET + +TEXT ·gemmUA7x8S(SB), NOSPLIT, $0-48 + MOVD kc+0(FP), R0 + MOVD a+8(FP), R1 + MOVD lda+16(FP), R2 + MOVD pb+24(FP), R13 + MOVD c+32(FP), R3 + MOVD ldc+40(FP), R4 + LSL $2, R2, R2 + LSL $2, R4, R4 + MOVD R1, R5 + ADD R2, R5, R6 + ADD R2, R6, R7 + ADD R2, R7, R8 + ADD R2, R8, R9 + ADD R2, R9, R10 + ADD R2, R10, R11 + VEOR V0.B16, V0.B16, V0.B16 + VEOR V1.B16, V1.B16, V1.B16 + VEOR V2.B16, V2.B16, V2.B16 + VEOR V3.B16, V3.B16, V3.B16 + VEOR V4.B16, V4.B16, V4.B16 + VEOR V5.B16, V5.B16, V5.B16 + VEOR V6.B16, V6.B16, V6.B16 + VEOR V7.B16, V7.B16, V7.B16 + VEOR V8.B16, V8.B16, V8.B16 + VEOR V9.B16, V9.B16, V9.B16 + VEOR V10.B16, V10.B16, V10.B16 + VEOR V11.B16, V11.B16, V11.B16 + VEOR V12.B16, V12.B16, V12.B16 + VEOR V13.B16, V13.B16, V13.B16 + LSR $2, R0, R14 + AND $3, R0, R15 + CBZ R14, uatail + uakloop: + VLD1.P 16(R5), [V20.S4] + VLD1.P 16(R6), [V21.S4] + VLD1.P 16(R7), [V22.S4] + VLD1.P 16(R8), [V23.S4] + VLD1.P 16(R9), [V24.S4] + VLD1.P 16(R10), [V25.S4] + VLD1.P 16(R11), [V26.S4] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] + WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] + WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] + WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] + WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] + WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] + WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] + WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] + WORD $0x4F981208 // FMLA V8.4S, V16.4S, V24.S[0] + WORD $0x4F981229 // FMLA V9.4S, V17.4S, V24.S[0] + WORD $0x4F99120A // FMLA V10.4S, V16.4S, V25.S[0] + WORD $0x4F99122B // FMLA V11.4S, V17.4S, V25.S[0] + WORD $0x4F9A120C // FMLA V12.4S, V16.4S, V26.S[0] + WORD $0x4F9A122D // FMLA V13.4S, V17.4S, V26.S[0] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] + WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] + WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] + WORD $0x4FB51223 // FMLA V3.4S, V17.4S, V21.S[1] + WORD $0x4FB61204 // FMLA V4.4S, V16.4S, V22.S[1] + WORD $0x4FB61225 // FMLA V5.4S, V17.4S, V22.S[1] + WORD $0x4FB71206 // FMLA V6.4S, V16.4S, V23.S[1] + WORD $0x4FB71227 // FMLA V7.4S, V17.4S, V23.S[1] + WORD $0x4FB81208 // FMLA V8.4S, V16.4S, V24.S[1] + WORD $0x4FB81229 // FMLA V9.4S, V17.4S, V24.S[1] + WORD $0x4FB9120A // FMLA V10.4S, V16.4S, V25.S[1] + WORD $0x4FB9122B // FMLA V11.4S, V17.4S, V25.S[1] + WORD $0x4FBA120C // FMLA V12.4S, V16.4S, V26.S[1] + WORD $0x4FBA122D // FMLA V13.4S, V17.4S, V26.S[1] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] + WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] + WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] + WORD $0x4F951A23 // FMLA V3.4S, V17.4S, V21.S[2] + WORD $0x4F961A04 // FMLA V4.4S, V16.4S, V22.S[2] + WORD $0x4F961A25 // FMLA V5.4S, V17.4S, V22.S[2] + WORD $0x4F971A06 // FMLA V6.4S, V16.4S, V23.S[2] + WORD $0x4F971A27 // FMLA V7.4S, V17.4S, V23.S[2] + WORD $0x4F981A08 // FMLA V8.4S, V16.4S, V24.S[2] + WORD $0x4F981A29 // FMLA V9.4S, V17.4S, V24.S[2] + WORD $0x4F991A0A // FMLA V10.4S, V16.4S, V25.S[2] + WORD $0x4F991A2B // FMLA V11.4S, V17.4S, V25.S[2] + WORD $0x4F9A1A0C // FMLA V12.4S, V16.4S, V26.S[2] + WORD $0x4F9A1A2D // FMLA V13.4S, V17.4S, V26.S[2] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] + WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] + WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] + WORD $0x4FB51A23 // FMLA V3.4S, V17.4S, V21.S[3] + WORD $0x4FB61A04 // FMLA V4.4S, V16.4S, V22.S[3] + WORD $0x4FB61A25 // FMLA V5.4S, V17.4S, V22.S[3] + WORD $0x4FB71A06 // FMLA V6.4S, V16.4S, V23.S[3] + WORD $0x4FB71A27 // FMLA V7.4S, V17.4S, V23.S[3] + WORD $0x4FB81A08 // FMLA V8.4S, V16.4S, V24.S[3] + WORD $0x4FB81A29 // FMLA V9.4S, V17.4S, V24.S[3] + WORD $0x4FB91A0A // FMLA V10.4S, V16.4S, V25.S[3] + WORD $0x4FB91A2B // FMLA V11.4S, V17.4S, V25.S[3] + WORD $0x4FBA1A0C // FMLA V12.4S, V16.4S, V26.S[3] + WORD $0x4FBA1A2D // FMLA V13.4S, V17.4S, V26.S[3] + SUB $1, R14 + CBNZ R14, uakloop + uatail: + CBZ R15, uastore + uatloop: + FMOVS.P 4(R5), F20 + FMOVS.P 4(R6), F21 + FMOVS.P 4(R7), F22 + FMOVS.P 4(R8), F23 + FMOVS.P 4(R9), F24 + FMOVS.P 4(R10), F25 + FMOVS.P 4(R11), F26 + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] + WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] + WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] + WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] + WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] + WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] + WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] + WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] + WORD $0x4F981208 // FMLA V8.4S, V16.4S, V24.S[0] + WORD $0x4F981229 // FMLA V9.4S, V17.4S, V24.S[0] + WORD $0x4F99120A // FMLA V10.4S, V16.4S, V25.S[0] + WORD $0x4F99122B // FMLA V11.4S, V17.4S, V25.S[0] + WORD $0x4F9A120C // FMLA V12.4S, V16.4S, V26.S[0] + WORD $0x4F9A122D // FMLA V13.4S, V17.4S, V26.S[0] + SUB $1, R15 + CBNZ R15, uatloop + uastore: + VLD1 (R3), [V16.S4, V17.S4] + VFADD V0.S4, V16.S4, V16.S4 + VFADD V1.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V2.S4, V16.S4, V16.S4 + VFADD V3.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V4.S4, V16.S4, V16.S4 + VFADD V5.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V6.S4, V16.S4, V16.S4 + VFADD V7.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V8.S4, V16.S4, V16.S4 + VFADD V9.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V10.S4, V16.S4, V16.S4 + VFADD V11.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V12.S4, V16.S4, V16.S4 + VFADD V13.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + RET + +TEXT ·gemmUA8x8S(SB), NOSPLIT, $0-48 + MOVD kc+0(FP), R0 + MOVD a+8(FP), R1 + MOVD lda+16(FP), R2 + MOVD pb+24(FP), R13 + MOVD c+32(FP), R3 + MOVD ldc+40(FP), R4 + LSL $2, R2, R2 + LSL $2, R4, R4 + MOVD R1, R5 + ADD R2, R5, R6 + ADD R2, R6, R7 + ADD R2, R7, R8 + ADD R2, R8, R9 + ADD R2, R9, R10 + ADD R2, R10, R11 + ADD R2, R11, R12 + VEOR V0.B16, V0.B16, V0.B16 + VEOR V1.B16, V1.B16, V1.B16 + VEOR V2.B16, V2.B16, V2.B16 + VEOR V3.B16, V3.B16, V3.B16 + VEOR V4.B16, V4.B16, V4.B16 + VEOR V5.B16, V5.B16, V5.B16 + VEOR V6.B16, V6.B16, V6.B16 + VEOR V7.B16, V7.B16, V7.B16 + VEOR V8.B16, V8.B16, V8.B16 + VEOR V9.B16, V9.B16, V9.B16 + VEOR V10.B16, V10.B16, V10.B16 + VEOR V11.B16, V11.B16, V11.B16 + VEOR V12.B16, V12.B16, V12.B16 + VEOR V13.B16, V13.B16, V13.B16 + VEOR V14.B16, V14.B16, V14.B16 + VEOR V15.B16, V15.B16, V15.B16 + LSR $2, R0, R14 + AND $3, R0, R15 + CBZ R14, uatail + uakloop: + VLD1.P 16(R5), [V20.S4] + VLD1.P 16(R6), [V21.S4] + VLD1.P 16(R7), [V22.S4] + VLD1.P 16(R8), [V23.S4] + VLD1.P 16(R9), [V24.S4] + VLD1.P 16(R10), [V25.S4] + VLD1.P 16(R11), [V26.S4] + VLD1.P 16(R12), [V27.S4] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] + WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] + WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] + WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] + WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] + WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] + WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] + WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] + WORD $0x4F981208 // FMLA V8.4S, V16.4S, V24.S[0] + WORD $0x4F981229 // FMLA V9.4S, V17.4S, V24.S[0] + WORD $0x4F99120A // FMLA V10.4S, V16.4S, V25.S[0] + WORD $0x4F99122B // FMLA V11.4S, V17.4S, V25.S[0] + WORD $0x4F9A120C // FMLA V12.4S, V16.4S, V26.S[0] + WORD $0x4F9A122D // FMLA V13.4S, V17.4S, V26.S[0] + WORD $0x4F9B120E // FMLA V14.4S, V16.4S, V27.S[0] + WORD $0x4F9B122F // FMLA V15.4S, V17.4S, V27.S[0] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] + WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] + WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] + WORD $0x4FB51223 // FMLA V3.4S, V17.4S, V21.S[1] + WORD $0x4FB61204 // FMLA V4.4S, V16.4S, V22.S[1] + WORD $0x4FB61225 // FMLA V5.4S, V17.4S, V22.S[1] + WORD $0x4FB71206 // FMLA V6.4S, V16.4S, V23.S[1] + WORD $0x4FB71227 // FMLA V7.4S, V17.4S, V23.S[1] + WORD $0x4FB81208 // FMLA V8.4S, V16.4S, V24.S[1] + WORD $0x4FB81229 // FMLA V9.4S, V17.4S, V24.S[1] + WORD $0x4FB9120A // FMLA V10.4S, V16.4S, V25.S[1] + WORD $0x4FB9122B // FMLA V11.4S, V17.4S, V25.S[1] + WORD $0x4FBA120C // FMLA V12.4S, V16.4S, V26.S[1] + WORD $0x4FBA122D // FMLA V13.4S, V17.4S, V26.S[1] + WORD $0x4FBB120E // FMLA V14.4S, V16.4S, V27.S[1] + WORD $0x4FBB122F // FMLA V15.4S, V17.4S, V27.S[1] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] + WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] + WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] + WORD $0x4F951A23 // FMLA V3.4S, V17.4S, V21.S[2] + WORD $0x4F961A04 // FMLA V4.4S, V16.4S, V22.S[2] + WORD $0x4F961A25 // FMLA V5.4S, V17.4S, V22.S[2] + WORD $0x4F971A06 // FMLA V6.4S, V16.4S, V23.S[2] + WORD $0x4F971A27 // FMLA V7.4S, V17.4S, V23.S[2] + WORD $0x4F981A08 // FMLA V8.4S, V16.4S, V24.S[2] + WORD $0x4F981A29 // FMLA V9.4S, V17.4S, V24.S[2] + WORD $0x4F991A0A // FMLA V10.4S, V16.4S, V25.S[2] + WORD $0x4F991A2B // FMLA V11.4S, V17.4S, V25.S[2] + WORD $0x4F9A1A0C // FMLA V12.4S, V16.4S, V26.S[2] + WORD $0x4F9A1A2D // FMLA V13.4S, V17.4S, V26.S[2] + WORD $0x4F9B1A0E // FMLA V14.4S, V16.4S, V27.S[2] + WORD $0x4F9B1A2F // FMLA V15.4S, V17.4S, V27.S[2] + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] + WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] + WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] + WORD $0x4FB51A23 // FMLA V3.4S, V17.4S, V21.S[3] + WORD $0x4FB61A04 // FMLA V4.4S, V16.4S, V22.S[3] + WORD $0x4FB61A25 // FMLA V5.4S, V17.4S, V22.S[3] + WORD $0x4FB71A06 // FMLA V6.4S, V16.4S, V23.S[3] + WORD $0x4FB71A27 // FMLA V7.4S, V17.4S, V23.S[3] + WORD $0x4FB81A08 // FMLA V8.4S, V16.4S, V24.S[3] + WORD $0x4FB81A29 // FMLA V9.4S, V17.4S, V24.S[3] + WORD $0x4FB91A0A // FMLA V10.4S, V16.4S, V25.S[3] + WORD $0x4FB91A2B // FMLA V11.4S, V17.4S, V25.S[3] + WORD $0x4FBA1A0C // FMLA V12.4S, V16.4S, V26.S[3] + WORD $0x4FBA1A2D // FMLA V13.4S, V17.4S, V26.S[3] + WORD $0x4FBB1A0E // FMLA V14.4S, V16.4S, V27.S[3] + WORD $0x4FBB1A2F // FMLA V15.4S, V17.4S, V27.S[3] + SUB $1, R14 + CBNZ R14, uakloop + uatail: + CBZ R15, uastore + uatloop: + FMOVS.P 4(R5), F20 + FMOVS.P 4(R6), F21 + FMOVS.P 4(R7), F22 + FMOVS.P 4(R8), F23 + FMOVS.P 4(R9), F24 + FMOVS.P 4(R10), F25 + FMOVS.P 4(R11), F26 + FMOVS.P 4(R12), F27 + VLD1.P 32(R13), [V16.S4, V17.S4] + WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] + WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] + WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] + WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] + WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] + WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] + WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] + WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] + WORD $0x4F981208 // FMLA V8.4S, V16.4S, V24.S[0] + WORD $0x4F981229 // FMLA V9.4S, V17.4S, V24.S[0] + WORD $0x4F99120A // FMLA V10.4S, V16.4S, V25.S[0] + WORD $0x4F99122B // FMLA V11.4S, V17.4S, V25.S[0] + WORD $0x4F9A120C // FMLA V12.4S, V16.4S, V26.S[0] + WORD $0x4F9A122D // FMLA V13.4S, V17.4S, V26.S[0] + WORD $0x4F9B120E // FMLA V14.4S, V16.4S, V27.S[0] + WORD $0x4F9B122F // FMLA V15.4S, V17.4S, V27.S[0] + SUB $1, R15 + CBNZ R15, uatloop + uastore: + VLD1 (R3), [V16.S4, V17.S4] + VFADD V0.S4, V16.S4, V16.S4 + VFADD V1.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V2.S4, V16.S4, V16.S4 + VFADD V3.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V4.S4, V16.S4, V16.S4 + VFADD V5.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V6.S4, V16.S4, V16.S4 + VFADD V7.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V8.S4, V16.S4, V16.S4 + VFADD V9.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V10.S4, V16.S4, V16.S4 + VFADD V11.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V12.S4, V16.S4, V16.S4 + VFADD V13.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V14.S4, V16.S4, V16.S4 + VFADD V15.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + RET + TEXT ·blocks8NEON(SB), NOSPLIT, $0-24 MOVD r+0(FP), R0 MOVD a+8(FP), R1 From b0870af6cc41f1dfc005b4dddf91054ceb4285e9 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Tue, 27 Oct 2026 12:51:43 +0100 Subject: [PATCH 2/2] docs: unpacked-A GEMM measurements and audit entry Co-Authored-By: Claude Opus 5.5 --- SECURITY.md | 9 +++++++++ docs/perf.md | 40 +++++++++++++++++++++++++++++++++++----- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 4eaa8e1..22bc605 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -66,6 +66,15 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none. ## Audit log +**2026-10-27 (v0.27.0).** The unpacked-A GEMM kernels (`gemmUA*` on amd64 +and arm64) read A in place: `mr` rows `lda` elements apart, `kc` elements +each. Their Go wrapper bounds-checks the whole reach, +`a[(mr-1)*lda+kc-1]`, the packed B panel and the C tile before the call. +`gemmBand` takes this path only for a unit column stride and a positive row +stride, so a reversed or broadcast view is still packed. The kernels are +fenced for every tile height and two row strides. A sabotaged kernel on each +architecture failed CI. + **2026-10-26 (v0.26.0).** The float32 GEMM kernels, `gemmMicro8x8S` (arm64) and `gemmMicro6x16FMA` (amd64, behind the FMA probe), read `kc·MR` and `kc·NR` packed floats and read and write one MR×NR tile of C. Their Go wrapper diff --git a/docs/perf.md b/docs/perf.md index cadd90d..3841545 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -1243,11 +1243,41 @@ element. Profiling had put it at 21% of a 64² float32 product. | Neoverse N1, float32 | −16.4% | −8.9% | ~ | | Neoverse N1, float64 | −9.5% | ~ | ~ | -**Open defect: products are still slower than OpenBLAS on one core**, by -1.1–1.3× at 1024² and up to 1.9× at 64². The gap is widest where the work -around the kernel weighs most: packing, edge tiles, and the result's -allocation and zeroing. The kernels for ppc64le, s390x, riscv64 and loong64 -are still to write. +### Reading A in place (2026-10-27) + +v0.27.0 no longer packs a row-major A (unit column stride, positive row +stride). Unpacked-A kernels read it in place: the "rv" gemmsup kernels of +BLIS (`kernels/haswell/3/sup/bli_gemmsup_rv_haswell_asm_s6x16m.c`). B stays +packed, so its last panel is still zero-padded. There is one kernel per tile +height, so a band's bottom rows never read past A's last row. + +- **amd64 (FMA):** each A value is broadcast straight from its row through + a scaled index (`VBROADCASTSS (SI)(R8*2)`), as BLIS does. +- **arm64:** the same idea, but broadcast loads (`VLD1R`) made it 26–78% + slower on a Neoverse N1, apparently because they compete with the FMAs for + the vector pipes. What shipped instead loads 4 floats (2 doubles) of each + row with one plain vector load, and the next k steps read them by lane with + the by-element FMLA. + +| `MatMul` vs v0.26.1 (`abbench`, 10 rounds) | 64² | 256² | 1024² | +|---|---:|---:|---:| +| Zen 3, float32 | −28% | −9.5% | −3.9% | +| Zen 3, float64 | −17% | −5.5% | ~ | +| Neoverse N1, float32 | −20% | −7.7% | ~ | +| Neoverse N1, float64 | −12% | −4.5% | ~ | + +Against NumPy 2.5.4 (OpenBLAS, one core), the ratio ours / NumPy: + +| | 64² | 256² | 1024² | +|---|---:|---:|---:| +| Zen 3 float32 | 1.18 | 1.13 | 1.04 | +| Zen 3 float64 | 1.16 | 1.03 | **0.97** | +| N1 float32 | 1.20 | 1.06 | 1.09 | +| N1 float64 | 1.18 | 1.11 | 1.18 | + +**Open defect: products are still up to 1.2× slower than OpenBLAS on one +core.** The exception is float64 at 1024² on Zen 3, where we are 3% faster. +The kernels for ppc64le, s390x, riscv64 and loong64 are still to write. **Open: the allocating operations.** `Add` is still 1.5–3.2× slower than NumPy's `a+b`, for float64 as well. The kernel is no longer the difference,