golang-image/vector/gen.go
Nigel Tao e78c45720c vector: add SIMD versions of xxxAccumulateMask.
name                              old time/op  new time/op  delta
GlyphAlphaLoose16Over-8           3.96µs ± 0%  3.64µs ± 1%   -8.08%   (p=0.000 n=8+10)
GlyphAlphaLoose16Src-8            3.64µs ± 0%  3.35µs ± 0%   -7.88%    (p=0.000 n=8+9)
GlyphAlphaLoose32Over-8           8.45µs ± 0%  6.74µs ± 0%  -20.22%    (p=0.000 n=8+9)
GlyphAlphaLoose32Src-8            7.24µs ± 0%  5.54µs ± 1%  -23.48%   (p=0.000 n=8+10)
GlyphAlphaLoose64Over-8           22.2µs ± 0%  17.4µs ± 0%  -21.67%    (p=0.000 n=9+9)
GlyphAlphaLoose64Src-8            17.6µs ± 1%  12.2µs ± 1%  -30.32%  (p=0.000 n=10+10)
GlyphAlphaLoose128Over-8          67.9µs ± 0%  53.3µs ± 1%  -21.53%  (p=0.000 n=10+10)
GlyphAlphaLoose128Src-8           48.2µs ± 0%  32.6µs ± 2%  -32.41%   (p=0.000 n=9+10)
GlyphAlphaLoose256Over-8           242µs ± 1%   187µs ± 1%  -22.96%    (p=0.000 n=9+9)
GlyphAlphaLoose256Src-8            163µs ± 0%   105µs ± 1%  -35.83%    (p=0.000 n=9+9)
GlyphRGBA16Over-8                 5.25µs ± 1%  4.95µs ± 0%   -5.78%    (p=0.000 n=9+9)
GlyphRGBA16Src-8                  4.72µs ± 0%  4.43µs ± 1%   -6.22%   (p=0.000 n=9+10)
GlyphRGBA32Over-8                 13.5µs ± 0%  11.9µs ± 1%  -12.19%   (p=0.000 n=9+10)
GlyphRGBA32Src-8                  11.5µs ± 1%   9.8µs ± 0%  -14.72%    (p=0.000 n=9+9)
GlyphRGBA64Over-8                 42.0µs ± 2%  36.9µs ± 1%  -12.19%  (p=0.000 n=10+10)
GlyphRGBA64Src-8                  34.1µs ± 1%  28.5µs ± 0%  -16.25%    (p=0.000 n=9+7)
GlyphRGBA128Over-8                 149µs ± 2%   133µs ± 1%  -10.24%   (p=0.000 n=10+9)
GlyphRGBA128Src-8                  115µs ± 1%    99µs ± 1%  -13.57%   (p=0.000 n=9+10)
GlyphRGBA256Over-8                 566µs ± 0%   511µs ± 1%   -9.85%   (p=0.000 n=9+10)
GlyphRGBA256Src-8                  435µs ± 0%   372µs ± 0%  -14.64%    (p=0.000 n=9+8)
GlyphNRGBA16Over-8                26.9µs ± 3%  26.0µs ± 3%   -3.55%   (p=0.000 n=10+9)
GlyphNRGBA16Src-8                 18.8µs ± 2%  18.4µs ± 2%   -2.21%   (p=0.000 n=9+10)
GlyphNRGBA32Over-8                99.1µs ± 2%  95.9µs ± 3%   -3.23%  (p=0.000 n=10+10)
GlyphNRGBA32Src-8                 65.6µs ± 3%  62.8µs ± 2%   -4.36%  (p=0.000 n=10+10)
GlyphNRGBA64Over-8                 376µs ± 4%   370µs ± 2%     ~     (p=0.063 n=10+10)
GlyphNRGBA64Src-8                  238µs ± 3%   233µs ± 1%   -2.21%   (p=0.000 n=9+10)
GlyphNRGBA128Over-8               1.52ms ± 2%  1.48ms ± 0%   -2.11%   (p=0.000 n=10+8)
GlyphNRGBA128Src-8                 951µs ± 3%   935µs ± 1%   -1.69%   (p=0.013 n=10+9)
GlyphNRGBA256Over-8               6.00ms ± 1%  5.87ms ± 3%   -2.12%  (p=0.002 n=10+10)
GlyphNRGBA256Src-8                3.94ms ± 2%  3.80ms ± 2%   -3.64%  (p=0.000 n=10+10)

A comparison of the non-SIMD and SIMD versions:

name                              time/op
FixedAccumulateMask16-8            237ns ± 0%
FixedAccumulateMaskSIMD16-8       80.0ns ± 1%
FloatingAccumulateMask16-8         413ns ± 2%
FloatingAccumulateMaskSIMD16-8     166ns ± 0%
FixedAccumulateMask64-8           3.42µs ± 0%
FixedAccumulateMaskSIMD64-8       1.09µs ± 0%
FloatingAccumulateMask64-8        6.92µs ± 0%
FloatingAccumulateMaskSIMD64-8    2.47µs ± 1%

Change-Id: Ib6980e5975ed2842ff2a372f76dd5f2e95c5526c
Reviewed-on: https://go-review.googlesource.com/30898
Reviewed-by: David Crawshaw <crawshaw@golang.org>
2016-10-14 04:48:47 +00:00

488 lines
13 KiB
Go

// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build ignore
package main
import (
"bytes"
"io/ioutil"
"log"
"strings"
"text/template"
)
const (
copyright = "" +
"// Copyright 2016 The Go Authors. All rights reserved.\n" +
"// Use of this source code is governed by a BSD-style\n" +
"// license that can be found in the LICENSE file.\n"
doNotEdit = "// generated by go run gen.go; DO NOT EDIT\n"
dashDashDash = "// --------"
)
func main() {
tmpl, err := ioutil.ReadFile("gen_acc_amd64.s.tmpl")
if err != nil {
log.Fatalf("ReadFile: %v", err)
}
if !bytes.HasPrefix(tmpl, []byte(copyright)) {
log.Fatal("source template did not start with the copyright header")
}
tmpl = tmpl[len(copyright):]
preamble := []byte(nil)
if i := bytes.Index(tmpl, []byte(dashDashDash)); i < 0 {
log.Fatalf("source template did not contain %q", dashDashDash)
} else {
preamble, tmpl = tmpl[:i], tmpl[i:]
}
t, err := template.New("").Parse(string(tmpl))
if err != nil {
log.Fatalf("Parse: %v", err)
}
out := bytes.NewBuffer(nil)
out.WriteString(doNotEdit)
out.Write(preamble)
for i, v := range instances {
if i != 0 {
out.WriteString("\n")
}
if strings.Contains(v.LoadArgs, "{{.ShortName}}") {
v.LoadArgs = strings.Replace(v.LoadArgs, "{{.ShortName}}", v.ShortName, -1)
}
if err := t.Execute(out, v); err != nil {
log.Fatalf("Execute(%q): %v", v.ShortName, err)
}
}
if err := ioutil.WriteFile("acc_amd64.s", out.Bytes(), 0666); err != nil {
log.Fatalf("WriteFile: %v", err)
}
}
var instances = []struct {
LongName string
ShortName string
FrameSize string
ArgsSize string
Args string
DstElemSize1 int
DstElemSize4 int
XMM3 string
XMM4 string
XMM5 string
XMM6 string
XMM8 string
XMM9 string
XMM10 string
LoadArgs string
Setup string
LoadXMMRegs string
Add string
ClampAndScale string
ConvertToInt32 string
Store4 string
Store1 string
}{{
LongName: "fixedAccumulateOpOver",
ShortName: "fxAccOpOver",
FrameSize: fxFrameSize,
ArgsSize: twoArgArgsSize,
Args: "dst []uint8, src []uint32",
DstElemSize1: 1 * sizeOfUint8,
DstElemSize4: 4 * sizeOfUint8,
XMM3: fxXMM3,
XMM4: fxXMM4,
XMM5: fxXMM5_65536,
XMM6: opOverXMM6,
XMM8: opOverXMM8,
XMM9: opOverXMM9,
XMM10: opOverXMM10,
LoadArgs: twoArgLoadArgs,
Setup: fxSetup,
LoadXMMRegs: fxLoadXMMRegs65536 + "\n" + opOverLoadXMMRegs,
Add: fxAdd,
ClampAndScale: fxClampAndScale65536,
ConvertToInt32: fxConvertToInt32,
Store4: opOverStore4,
Store1: opOverStore1,
}, {
LongName: "fixedAccumulateOpSrc",
ShortName: "fxAccOpSrc",
FrameSize: fxFrameSize,
ArgsSize: twoArgArgsSize,
Args: "dst []uint8, src []uint32",
DstElemSize1: 1 * sizeOfUint8,
DstElemSize4: 4 * sizeOfUint8,
XMM3: fxXMM3,
XMM4: fxXMM4,
XMM5: fxXMM5_256,
XMM6: opSrcXMM6,
XMM8: opSrcXMM8,
XMM9: opSrcXMM9,
XMM10: opSrcXMM10,
LoadArgs: twoArgLoadArgs,
Setup: fxSetup,
LoadXMMRegs: fxLoadXMMRegs256 + "\n" + opSrcLoadXMMRegs,
Add: fxAdd,
ClampAndScale: fxClampAndScale256,
ConvertToInt32: fxConvertToInt32,
Store4: opSrcStore4,
Store1: opSrcStore1,
}, {
LongName: "fixedAccumulateMask",
ShortName: "fxAccMask",
FrameSize: fxFrameSize,
ArgsSize: oneArgArgsSize,
Args: "buf []uint32",
DstElemSize1: 1 * sizeOfUint32,
DstElemSize4: 4 * sizeOfUint32,
XMM3: fxXMM3,
XMM4: fxXMM4,
XMM5: fxXMM5_65536,
XMM6: maskXMM6,
XMM8: maskXMM8,
XMM9: maskXMM9,
XMM10: maskXMM10,
LoadArgs: oneArgLoadArgs,
Setup: fxSetup,
LoadXMMRegs: fxLoadXMMRegs65536 + "\n" + maskLoadXMMRegs,
Add: fxAdd,
ClampAndScale: fxClampAndScale65536,
ConvertToInt32: fxConvertToInt32,
Store4: maskStore4,
Store1: maskStore1,
}, {
LongName: "floatingAccumulateOpOver",
ShortName: "flAccOpOver",
FrameSize: flFrameSize,
ArgsSize: twoArgArgsSize,
Args: "dst []uint8, src []float32",
DstElemSize1: 1 * sizeOfUint8,
DstElemSize4: 4 * sizeOfUint8,
XMM3: flXMM3_65536,
XMM4: flXMM4,
XMM5: flXMM5,
XMM6: opOverXMM6,
XMM8: opOverXMM8,
XMM9: opOverXMM9,
XMM10: opOverXMM10,
LoadArgs: twoArgLoadArgs,
Setup: flSetup,
LoadXMMRegs: flLoadXMMRegs65536 + "\n" + opOverLoadXMMRegs,
Add: flAdd,
ClampAndScale: flClampAndScale65536,
ConvertToInt32: flConvertToInt32,
Store4: opOverStore4,
Store1: opOverStore1,
}, {
LongName: "floatingAccumulateOpSrc",
ShortName: "flAccOpSrc",
FrameSize: flFrameSize,
ArgsSize: twoArgArgsSize,
Args: "dst []uint8, src []float32",
DstElemSize1: 1 * sizeOfUint8,
DstElemSize4: 4 * sizeOfUint8,
XMM3: flXMM3_256,
XMM4: flXMM4,
XMM5: flXMM5,
XMM6: opSrcXMM6,
XMM8: opSrcXMM8,
XMM9: opSrcXMM9,
XMM10: opSrcXMM10,
LoadArgs: twoArgLoadArgs,
Setup: flSetup,
LoadXMMRegs: flLoadXMMRegs256 + "\n" + opSrcLoadXMMRegs,
Add: flAdd,
ClampAndScale: flClampAndScale256,
ConvertToInt32: flConvertToInt32,
Store4: opSrcStore4,
Store1: opSrcStore1,
}, {
LongName: "floatingAccumulateMask",
ShortName: "flAccMask",
FrameSize: flFrameSize,
ArgsSize: twoArgArgsSize,
Args: "dst []uint32, src []float32",
DstElemSize1: 1 * sizeOfUint32,
DstElemSize4: 4 * sizeOfUint32,
XMM3: flXMM3_65536,
XMM4: flXMM4,
XMM5: flXMM5,
XMM6: maskXMM6,
XMM8: maskXMM8,
XMM9: maskXMM9,
XMM10: maskXMM10,
LoadArgs: twoArgLoadArgs,
Setup: flSetup,
LoadXMMRegs: flLoadXMMRegs65536 + "\n" + maskLoadXMMRegs,
Add: flAdd,
ClampAndScale: flClampAndScale65536,
ConvertToInt32: flConvertToInt32,
Store4: maskStore4,
Store1: maskStore1,
}}
const (
fxFrameSize = `0`
flFrameSize = `8`
oneArgArgsSize = `24`
twoArgArgsSize = `48`
sizeOfUint8 = 1
sizeOfUint32 = 4
fxXMM3 = `-`
flXMM3_256 = `flAlmost256`
flXMM3_65536 = `flAlmost65536`
fxXMM4 = `-`
flXMM4 = `flOne`
fxXMM5_256 = `fxAlmost256`
fxXMM5_65536 = `fxAlmost65536`
flXMM5 = `flSignMask`
oneArgLoadArgs = `
MOVQ buf_base+0(FP), DI
MOVQ buf_len+8(FP), BX
MOVQ buf_base+0(FP), SI
MOVQ buf_len+8(FP), R10
`
twoArgLoadArgs = `
MOVQ dst_base+0(FP), DI
MOVQ dst_len+8(FP), BX
MOVQ src_base+24(FP), SI
MOVQ src_len+32(FP), R10
// Sanity check that len(dst) >= len(src).
CMPQ BX, R10
JLT {{.ShortName}}End
`
fxSetup = ``
flSetup = `
// Prepare to set MXCSR bits 13 and 14, so that the CVTPS2PL below is
// "Round To Zero".
STMXCSR mxcsrOrig-8(SP)
MOVL mxcsrOrig-8(SP), AX
ORL $0x6000, AX
MOVL AX, mxcsrNew-4(SP)
`
fxLoadXMMRegs256 = `
// fxAlmost256 := XMM(0x000000ff repeated four times) // Maximum of an uint8.
MOVOU fxAlmost256<>(SB), X5
`
fxLoadXMMRegs65536 = `
// fxAlmost65536 := XMM(0x0000ffff repeated four times) // Maximum of an uint16.
MOVOU fxAlmost65536<>(SB), X5
`
flLoadXMMRegs256 = `
// flAlmost256 := XMM(0x437fffff repeated four times) // 255.99998 as a float32.
// flOne := XMM(0x3f800000 repeated four times) // 1 as a float32.
// flSignMask := XMM(0x7fffffff repeated four times) // All but the sign bit of a float32.
MOVOU flAlmost256<>(SB), X3
MOVOU flOne<>(SB), X4
MOVOU flSignMask<>(SB), X5
`
flLoadXMMRegs65536 = `
// flAlmost65536 := XMM(0x477fffff repeated four times) // 255.99998 * 256 as a float32.
// flOne := XMM(0x3f800000 repeated four times) // 1 as a float32.
// flSignMask := XMM(0x7fffffff repeated four times) // All but the sign bit of a float32.
MOVOU flAlmost65536<>(SB), X3
MOVOU flOne<>(SB), X4
MOVOU flSignMask<>(SB), X5
`
fxAdd = `PADDD`
flAdd = `ADDPS`
fxClampAndScale256 = `
// y = abs(x)
// y >>= 12 // Shift by 2*ϕ - 8.
// y = min(y, fxAlmost256)
//
// pabsd %xmm1,%xmm2
// psrld $0xc,%xmm2
// pminud %xmm5,%xmm2
//
// Hopefully we'll get these opcode mnemonics into the assembler for Go
// 1.8. https://golang.org/issue/16007 isn't exactly the same thing, but
// it's similar.
BYTE $0x66; BYTE $0x0f; BYTE $0x38; BYTE $0x1e; BYTE $0xd1
BYTE $0x66; BYTE $0x0f; BYTE $0x72; BYTE $0xd2; BYTE $0x0c
BYTE $0x66; BYTE $0x0f; BYTE $0x38; BYTE $0x3b; BYTE $0xd5
`
fxClampAndScale65536 = `
// y = abs(x)
// y >>= 4 // Shift by 2*ϕ - 16.
// y = min(y, fxAlmost65536)
//
// pabsd %xmm1,%xmm2
// psrld $0x4,%xmm2
// pminud %xmm5,%xmm2
//
// Hopefully we'll get these opcode mnemonics into the assembler for Go
// 1.8. https://golang.org/issue/16007 isn't exactly the same thing, but
// it's similar.
BYTE $0x66; BYTE $0x0f; BYTE $0x38; BYTE $0x1e; BYTE $0xd1
BYTE $0x66; BYTE $0x0f; BYTE $0x72; BYTE $0xd2; BYTE $0x04
BYTE $0x66; BYTE $0x0f; BYTE $0x38; BYTE $0x3b; BYTE $0xd5
`
flClampAndScale256 = `
// y = x & flSignMask
// y = min(y, flOne)
// y = mul(y, flAlmost256)
MOVOU X5, X2
ANDPS X1, X2
MINPS X4, X2
MULPS X3, X2
`
flClampAndScale65536 = `
// y = x & flSignMask
// y = min(y, flOne)
// y = mul(y, flAlmost65536)
MOVOU X5, X2
ANDPS X1, X2
MINPS X4, X2
MULPS X3, X2
`
fxConvertToInt32 = `
// z = convertToInt32(y)
// No-op.
`
flConvertToInt32 = `
// z = convertToInt32(y)
LDMXCSR mxcsrNew-4(SP)
CVTPS2PL X2, X2
LDMXCSR mxcsrOrig-8(SP)
`
opOverStore4 = `
// Blend over the dst's prior value. SIMD for i in 0..3:
//
// dstA := uint32(dst[i]) * 0x101
// maskA := z@i
// outA := dstA*(0xffff-maskA)/0xffff + maskA
// dst[i] = uint8(outA >> 8)
//
// First, set X0 to dstA*(0xfff-maskA).
MOVL (DI), X0
PSHUFB X8, X0
MOVOU X9, X11
PSUBL X2, X11
PMULLD X11, X0
// We implement uint32 division by 0xffff as multiplication by a magic
// constant (0x800080001) and then a shift by a magic constant (47).
// See TestDivideByFFFF for a justification.
//
// That multiplication widens from uint32 to uint64, so we have to
// duplicate and shift our four uint32s from one XMM register (X0) to
// two XMM registers (X0 and X11).
//
// Move the second and fourth uint32s in X0 to be the first and third
// uint32s in X11.
MOVOU X0, X11
PSRLQ $32, X11
// Multiply by magic, shift by magic.
//
// pmuludq %xmm10,%xmm0
// pmuludq %xmm10,%xmm11
BYTE $0x66; BYTE $0x41; BYTE $0x0f; BYTE $0xf4; BYTE $0xc2
BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0xf4; BYTE $0xda
PSRLQ $47, X0
PSRLQ $47, X11
// Merge the two registers back to one, X11.
PSLLQ $32, X11
XORPS X0, X11
// Add maskA, shift from 16 bit color to 8 bit color.
PADDD X11, X2
PSRLQ $8, X2
// As per opSrcStore4, shuffle and copy the low 4 bytes.
PSHUFB X6, X2
MOVL X2, (DI)
`
opSrcStore4 = `
// z = shuffleTheLowBytesOfEach4ByteElement(z)
// copy(dst[:4], low4BytesOf(z))
PSHUFB X6, X2
MOVL X2, (DI)
`
maskStore4 = `
// copy(dst[:4], z)
MOVOU X2, (DI)
`
opOverStore1 = `
// Blend over the dst's prior value.
//
// dstA := uint32(dst[0]) * 0x101
// maskA := z
// outA := dstA*(0xffff-maskA)/0xffff + maskA
// dst[0] = uint8(outA >> 8)
MOVBLZX (DI), R12
IMULL $0x101, R12
MOVL X2, R13
MOVL $0xffff, AX
SUBL R13, AX
MULL R12 // MULL's implicit arg is AX, and the result is stored in DX:AX.
MOVL $0x80008001, BX // Divide by 0xffff is to first multiply by a magic constant...
MULL BX // MULL's implicit arg is AX, and the result is stored in DX:AX.
SHRL $15, DX // ...and then shift by another magic constant (47 - 32 = 15).
ADDL DX, R13
SHRL $8, R13
MOVB R13, (DI)
`
opSrcStore1 = `
// dst[0] = uint8(z)
MOVL X2, BX
MOVB BX, (DI)
`
maskStore1 = `
// dst[0] = uint32(z)
MOVL X2, (DI)
`
opOverXMM6 = `gather`
opSrcXMM6 = `gather`
maskXMM6 = `-`
opOverXMM8 = `scatterAndMulBy0x101`
opSrcXMM8 = `-`
maskXMM8 = `-`
opOverXMM9 = `fxAlmost65536`
opSrcXMM9 = `-`
maskXMM9 = `-`
opOverXMM10 = `inverseFFFF`
opSrcXMM10 = `-`
maskXMM10 = `-`
opOverLoadXMMRegs = `
// gather := XMM(see above) // PSHUFB shuffle mask.
// scatterAndMulBy0x101 := XMM(see above) // PSHUFB shuffle mask.
// fxAlmost65536 := XMM(0x0000ffff repeated four times) // 0xffff.
// inverseFFFF := XMM(0x80008001 repeated four times) // Magic constant for dividing by 0xffff.
MOVOU gather<>(SB), X6
MOVOU scatterAndMulBy0x101<>(SB), X8
MOVOU fxAlmost65536<>(SB), X9
MOVOU inverseFFFF<>(SB), X10
`
opSrcLoadXMMRegs = `
// gather := XMM(see above) // PSHUFB shuffle mask.
MOVOU gather<>(SB), X6
`
maskLoadXMMRegs = ``
)