golang-image/vector/gen_acc_amd64.s.tmpl
Nigel Tao 93fad3647f vector: remove a shift from the inner loop.
PSHUFB already lets us pick certain bytes out of an XMM register.
There's no need to shift by 8 bits (1 byte) beforehand.

The code generator is simpler. There's also a small win on the
benchmarks, especially for FixedAccumulateOpOverSIMD.

name                              old time/op  new time/op  delta
FixedAccumulateOpOverSIMD16-8      183ns ± 0%   174ns ± 0%  -4.92%    (p=0.000 n=8+8)
FixedAccumulateOpSrcSIMD16-8      87.0ns ± 1%  86.3ns ± 0%  -0.77%   (p=0.000 n=10+9)
FixedAccumulateMaskSIMD16-8       80.4ns ± 1%  81.2ns ± 1%  +1.01%  (p=0.000 n=10+10)
FloatingAccumulateOpOverSIMD16-8   250ns ± 1%   244ns ± 0%  -2.39%   (p=0.000 n=10+8)
FloatingAccumulateOpSrcSIMD16-8    176ns ± 1%   176ns ± 0%    ~      (p=0.142 n=10+8)
FloatingAccumulateMaskSIMD16-8     167ns ± 0%   167ns ± 0%    ~      (p=0.137 n=8+10)
FixedAccumulateOpOverSIMD64-8     2.73µs ± 1%  2.58µs ± 0%  -5.36%   (p=0.000 n=10+7)
FixedAccumulateOpSrcSIMD64-8      1.18µs ± 1%  1.17µs ± 0%  -0.33%   (p=0.003 n=10+9)
FixedAccumulateMaskSIMD64-8       1.09µs ± 0%  1.09µs ± 0%  -0.17%    (p=0.047 n=9+9)
FloatingAccumulateOpOverSIMD64-8  3.67µs ± 0%  3.61µs ± 1%  -1.47%   (p=0.000 n=7+10)
FloatingAccumulateOpSrcSIMD64-8   2.60µs ± 0%  2.61µs ± 0%  +0.19%    (p=0.003 n=8+8)
FloatingAccumulateMaskSIMD64-8    2.47µs ± 0%  2.46µs ± 0%    ~      (p=0.162 n=10+9)
GlyphAlpha16Over-8                2.99µs ± 0%  2.98µs ± 1%  -0.50%   (p=0.021 n=9+10)
GlyphAlpha16Src-8                 2.89µs ± 1%  2.89µs ± 0%    ~     (p=0.381 n=10+10)
GlyphAlpha32Over-8                4.53µs ± 0%  4.50µs ± 0%  -0.83%  (p=0.000 n=10+10)
GlyphAlpha32Src-8                 4.14µs ± 0%  4.13µs ± 0%  -0.21%   (p=0.026 n=9+10)
GlyphAlpha64Over-8                8.97µs ± 1%  8.80µs ± 0%  -1.85%   (p=0.000 n=10+9)
GlyphAlpha64Src-8                 7.42µs ± 1%  7.39µs ± 0%  -0.45%  (p=0.011 n=10+10)
GlyphAlpha128Over-8               21.8µs ± 0%  21.2µs ± 0%  -2.91%    (p=0.000 n=9+9)
GlyphAlpha128Src-8                15.6µs ± 0%  15.6µs ± 0%    ~      (p=0.982 n=10+7)
GlyphAlpha256Over-8               66.3µs ± 1%  63.7µs ± 0%  -3.84%   (p=0.000 n=10+9)
GlyphAlpha256Src-8                41.2µs ± 1%  41.2µs ± 1%    ~     (p=1.000 n=10+10)
GlyphRGBA16Over-8                 4.75µs ± 0%  4.75µs ± 1%    ~      (p=0.735 n=9+10)
GlyphRGBA16Src-8                  4.20µs ± 0%  4.20µs ± 0%    ~       (p=0.503 n=8+8)
GlyphRGBA32Over-8                 11.4µs ± 0%  11.4µs ± 0%    ~       (p=0.119 n=9+9)
GlyphRGBA32Src-8                  9.34µs ± 1%  9.32µs ± 0%    ~       (p=0.062 n=9+8)
GlyphRGBA64Over-8                 36.0µs ± 0%  36.1µs ± 0%    ~       (p=0.209 n=8+9)
GlyphRGBA64Src-8                  27.9µs ± 1%  27.8µs ± 0%    ~     (p=0.796 n=10+10)
GlyphRGBA128Over-8                 131µs ± 0%   131µs ± 0%    ~       (p=0.931 n=9+9)
GlyphRGBA128Src-8                 97.9µs ± 0%  97.7µs ± 1%    ~      (p=0.053 n=9+10)
GlyphRGBA256Over-8                 503µs ± 0%   503µs ± 1%    ~      (p=0.274 n=8+10)
GlyphRGBA256Src-8                  370µs ± 0%   369µs ± 0%    ~      (p=0.497 n=9+10)

Change-Id: I56651e70b258792b83ea2a74904756243c88bef4
Reviewed-on: https://go-review.googlesource.com/31537
Reviewed-by: David Crawshaw <crawshaw@golang.org>
2016-10-22 21:39:22 +00:00

171 lines
4.0 KiB
Cheetah

// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !appengine
// +build gc
// +build !noasm
#include "textflag.h"
// fl is short for floating point math. fx is short for fixed point math.
DATA flAlmost65536<>+0x00(SB)/8, $0x477fffff477fffff
DATA flAlmost65536<>+0x08(SB)/8, $0x477fffff477fffff
DATA flOne<>+0x00(SB)/8, $0x3f8000003f800000
DATA flOne<>+0x08(SB)/8, $0x3f8000003f800000
DATA flSignMask<>+0x00(SB)/8, $0x7fffffff7fffffff
DATA flSignMask<>+0x08(SB)/8, $0x7fffffff7fffffff
// scatterAndMulBy0x101 is a PSHUFB mask that brings the low four bytes of an
// XMM register to the low byte of that register's four uint32 values. It
// duplicates those bytes, effectively multiplying each uint32 by 0x101.
//
// It transforms a little-endian 16-byte XMM value from
// ijkl????????????
// to
// ii00jj00kk00ll00
DATA scatterAndMulBy0x101<>+0x00(SB)/8, $0x8080010180800000
DATA scatterAndMulBy0x101<>+0x08(SB)/8, $0x8080030380800202
// gather is a PSHUFB mask that brings the second-lowest byte of the XMM
// register's four uint32 values to the low four bytes of that register.
//
// It transforms a little-endian 16-byte XMM value from
// ?i???j???k???l??
// to
// ijkl000000000000
DATA gather<>+0x00(SB)/8, $0x808080800d090501
DATA gather<>+0x08(SB)/8, $0x8080808080808080
DATA fxAlmost65536<>+0x00(SB)/8, $0x0000ffff0000ffff
DATA fxAlmost65536<>+0x08(SB)/8, $0x0000ffff0000ffff
DATA inverseFFFF<>+0x00(SB)/8, $0x8000800180008001
DATA inverseFFFF<>+0x08(SB)/8, $0x8000800180008001
GLOBL flAlmost65536<>(SB), (NOPTR+RODATA), $16
GLOBL flOne<>(SB), (NOPTR+RODATA), $16
GLOBL flSignMask<>(SB), (NOPTR+RODATA), $16
GLOBL scatterAndMulBy0x101<>(SB), (NOPTR+RODATA), $16
GLOBL gather<>(SB), (NOPTR+RODATA), $16
GLOBL fxAlmost65536<>(SB), (NOPTR+RODATA), $16
GLOBL inverseFFFF<>(SB), (NOPTR+RODATA), $16
// func haveSSE4_1() bool
TEXT ·haveSSE4_1(SB), NOSPLIT, $0
MOVQ $1, AX
CPUID
SHRQ $19, CX
ANDQ $1, CX
MOVB CX, ret+0(FP)
RET
// ----------------------------------------------------------------------------
// func {{.LongName}}SIMD({{.Args}})
//
// XMM registers. Variable names are per
// https://github.com/google/font-rs/blob/master/src/accumulate.c
//
// xmm0 scratch
// xmm1 x
// xmm2 y, z
// xmm3 {{.XMM3}}
// xmm4 {{.XMM4}}
// xmm5 {{.XMM5}}
// xmm6 {{.XMM6}}
// xmm7 offset
// xmm8 {{.XMM8}}
// xmm9 {{.XMM9}}
// xmm10 {{.XMM10}}
TEXT ·{{.LongName}}SIMD(SB), NOSPLIT, ${{.FrameSize}}-{{.ArgsSize}}
{{.LoadArgs}}
// R10 = len(src) &^ 3
// R11 = len(src)
MOVQ R10, R11
ANDQ $-4, R10
{{.Setup}}
{{.LoadXMMRegs}}
// offset := XMM(0x00000000 repeated four times) // Cumulative sum.
XORPS X7, X7
// i := 0
MOVQ $0, R9
{{.ShortName}}Loop4:
// for i < (len(src) &^ 3)
CMPQ R9, R10
JAE {{.ShortName}}Loop1
// x = XMM(s0, s1, s2, s3)
//
// Where s0 is src[i+0], s1 is src[i+1], etc.
MOVOU (SI), X1
// scratch = XMM(0, s0, s1, s2)
// x += scratch // yields x == XMM(s0, s0+s1, s1+s2, s2+s3)
MOVOU X1, X0
PSLLO $4, X0
{{.Add}} X0, X1
// scratch = XMM(0, 0, 0, 0)
// scratch = XMM(scratch@0, scratch@0, x@0, x@1) // yields scratch == XMM(0, 0, s0, s0+s1)
// x += scratch // yields x == XMM(s0, s0+s1, s0+s1+s2, s0+s1+s2+s3)
XORPS X0, X0
SHUFPS $0x40, X1, X0
{{.Add}} X0, X1
// x += offset
{{.Add}} X7, X1
{{.ClampAndScale}}
{{.ConvertToInt32}}
{{.Store4}}
// offset = XMM(x@3, x@3, x@3, x@3)
MOVOU X1, X7
SHUFPS $0xff, X1, X7
// i += 4
// dst = dst[4:]
// src = src[4:]
ADDQ $4, R9
ADDQ ${{.DstElemSize4}}, DI
ADDQ $16, SI
JMP {{.ShortName}}Loop4
{{.ShortName}}Loop1:
// for i < len(src)
CMPQ R9, R11
JAE {{.ShortName}}End
// x = src[i] + offset
MOVL (SI), X1
{{.Add}} X7, X1
{{.ClampAndScale}}
{{.ConvertToInt32}}
{{.Store1}}
// offset = x
MOVOU X1, X7
// i += 1
// dst = dst[1:]
// src = src[1:]
ADDQ $1, R9
ADDQ ${{.DstElemSize1}}, DI
ADDQ $4, SI
JMP {{.ShortName}}Loop1
{{.ShortName}}End:
RET