746988e7a2
name old time/op new time/op delta GlyphAlpha16Src-8 3.37µs ± 0% 3.07µs ± 1% -8.86% (p=0.000 n=9+9) GlyphAlpha32Src-8 6.01µs ± 1% 4.55µs ± 0% -24.28% (p=0.000 n=10+9) GlyphAlpha64Src-8 13.2µs ± 0% 8.1µs ± 0% -38.69% (p=0.000 n=10+9) GlyphAlpha128Src-8 32.9µs ± 0% 16.9µs ± 0% -48.85% (p=0.000 n=10+9) GlyphAlpha256Src-8 98.0µs ± 0% 43.6µs ± 1% -55.50% (p=0.000 n=10+10) A comparison of the non-SIMD and SIMD versions: name time/op FixedAccumulateOpSrc16-8 368ns ± 0% FixedAccumulateOpSrcSIMD16-8 86.8ns ± 1% FloatingAccumulateOpSrc16-8 434ns ± 0% FloatingAccumulateOpSrcSIMD16-8 119ns ± 0% FixedAccumulateOpSrc64-8 6.12µs ± 0% FixedAccumulateOpSrcSIMD64-8 1.17µs ± 0% FloatingAccumulateOpSrc64-8 7.15µs ± 0% FloatingAccumulateOpSrcSIMD64-8 1.68µs ± 1% Change-Id: I58e5c7a3ecd12e536aab8e765e94275453d0eac8 Reviewed-on: https://go-review.googlesource.com/30431 Reviewed-by: David Crawshaw <crawshaw@golang.org>
322 lines
7.4 KiB
ArmAsm
322 lines
7.4 KiB
ArmAsm
// Copyright 2016 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
// +build !appengine
|
|
// +build gc
|
|
// +build !noasm
|
|
|
|
#include "textflag.h"
|
|
|
|
// fl is short for floating point math. fx is short for fixed point math.
|
|
|
|
DATA flAlmost256<>+0x00(SB)/8, $0x437fffff437fffff
|
|
DATA flAlmost256<>+0x08(SB)/8, $0x437fffff437fffff
|
|
DATA flOnes<>+0x00(SB)/8, $0x3f8000003f800000
|
|
DATA flOnes<>+0x08(SB)/8, $0x3f8000003f800000
|
|
DATA flSignMask<>+0x00(SB)/8, $0x7fffffff7fffffff
|
|
DATA flSignMask<>+0x08(SB)/8, $0x7fffffff7fffffff
|
|
DATA shuffleMask<>+0x00(SB)/8, $0x0c0804000c080400
|
|
DATA shuffleMask<>+0x08(SB)/8, $0x0c0804000c080400
|
|
DATA fxAlmost256<>+0x00(SB)/8, $0x000000ff000000ff
|
|
DATA fxAlmost256<>+0x08(SB)/8, $0x000000ff000000ff
|
|
|
|
GLOBL flAlmost256<>(SB), (NOPTR+RODATA), $16
|
|
GLOBL flOnes<>(SB), (NOPTR+RODATA), $16
|
|
GLOBL flSignMask<>(SB), (NOPTR+RODATA), $16
|
|
GLOBL shuffleMask<>(SB), (NOPTR+RODATA), $16
|
|
GLOBL fxAlmost256<>(SB), (NOPTR+RODATA), $16
|
|
|
|
// func haveSSE4_1() bool
|
|
TEXT ·haveSSE4_1(SB), NOSPLIT, $0
|
|
MOVQ $1, AX
|
|
CPUID
|
|
SHRQ $19, CX
|
|
ANDQ $1, CX
|
|
MOVB CX, ret+0(FP)
|
|
RET
|
|
|
|
// ----------------------------------------------------------------------------
|
|
|
|
// func fixedAccumulateOpSrcSIMD(dst []uint8, src []uint32)
|
|
//
|
|
// XMM registers. Variable names are per
|
|
// https://github.com/google/font-rs/blob/master/src/accumulate.c
|
|
//
|
|
// xmm0 scratch
|
|
// xmm1 x
|
|
// xmm2 y, z
|
|
// xmm3 -
|
|
// xmm4 -
|
|
// xmm5 fxAlmost256
|
|
// xmm6 shuffleMask
|
|
// xmm7 offset
|
|
TEXT ·fixedAccumulateOpSrcSIMD(SB), NOSPLIT, $0-48
|
|
MOVQ dst_base+0(FP), DI
|
|
MOVQ dst_len+8(FP), BX
|
|
MOVQ src_base+24(FP), SI
|
|
MOVQ src_len+32(FP), CX
|
|
|
|
// Sanity check that len(dst) >= len(src).
|
|
CMPQ BX, CX
|
|
JLT fxAccOpSrcEnd
|
|
|
|
// CX = len(src) &^ 3
|
|
// DX = len(src)
|
|
MOVQ CX, DX
|
|
ANDQ $-4, CX
|
|
|
|
// fxAlmost256 := XMM(0x000000ff repeated four times) // Maximum of an uint8.
|
|
// shuffleMask := XMM(0x0c080400 repeated four times) // PSHUFB shuffle mask.
|
|
// offset := XMM(0x00000000 repeated four times) // Cumulative sum.
|
|
MOVOU fxAlmost256<>(SB), X5
|
|
MOVOU shuffleMask<>(SB), X6
|
|
XORPS X7, X7
|
|
|
|
// i := 0
|
|
MOVQ $0, AX
|
|
|
|
fxAccOpSrcLoop4:
|
|
// for i < (len(src) &^ 3)
|
|
CMPQ AX, CX
|
|
JAE fxAccOpSrcLoop1
|
|
|
|
// x = XMM(s0, s1, s2, s3)
|
|
//
|
|
// Where s0 is src[i+0], s1 is src[i+1], etc.
|
|
MOVOU (SI), X1
|
|
|
|
// scratch = XMM(0, s0, s1, s2)
|
|
// x += scratch // yields x == XMM(s0, s0+s1, s1+s2, s2+s3)
|
|
MOVOU X1, X0
|
|
PSLLO $4, X0
|
|
PADDD X0, X1
|
|
|
|
// scratch = XMM(0, 0, 0, 0)
|
|
// scratch = XMM(scratch@0, scratch@0, x@0, x@1) // yields scratch == XMM(0, 0, s0, s0+s1)
|
|
// x += scratch // yields x == XMM(s0, s0+s1, s0+s1+s2, s0+s1+s2+s3)
|
|
XORPS X0, X0
|
|
SHUFPS $0x40, X1, X0
|
|
PADDD X0, X1
|
|
|
|
// x += offset
|
|
PADDD X7, X1
|
|
|
|
// y = abs(x)
|
|
// y >>= 12 // Shift by 2*ϕ - 8.
|
|
// y = min(y, fxAlmost256)
|
|
//
|
|
// pabsd %xmm1,%xmm2
|
|
// psrld $0xc,%xmm2
|
|
// pminud %xmm5,%xmm2
|
|
//
|
|
// Hopefully we'll get these opcode mnemonics into the assembler for Go
|
|
// 1.8. https://golang.org/issue/16007 isn't exactly the same thing, but
|
|
// it's similar.
|
|
BYTE $0x66; BYTE $0x0f; BYTE $0x38; BYTE $0x1e; BYTE $0xd1
|
|
BYTE $0x66; BYTE $0x0f; BYTE $0x72; BYTE $0xd2; BYTE $0x0c
|
|
BYTE $0x66; BYTE $0x0f; BYTE $0x38; BYTE $0x3b; BYTE $0xd5
|
|
|
|
// z = shuffleTheLowBytesOfEach4ByteElement(y)
|
|
// copy(dst[:4], low4BytesOf(z))
|
|
PSHUFB X6, X2
|
|
MOVL X2, (DI)
|
|
|
|
// offset = XMM(x@3, x@3, x@3, x@3)
|
|
MOVOU X1, X7
|
|
SHUFPS $0xff, X1, X7
|
|
|
|
// i += 4
|
|
// dst = dst[4:]
|
|
// src = src[4:]
|
|
ADDQ $4, AX
|
|
ADDQ $4, DI
|
|
ADDQ $16, SI
|
|
JMP fxAccOpSrcLoop4
|
|
|
|
fxAccOpSrcLoop1:
|
|
// for i < len(src)
|
|
CMPQ AX, DX
|
|
JAE fxAccOpSrcEnd
|
|
|
|
// x = src[i] + offset
|
|
MOVL (SI), X1
|
|
PADDD X7, X1
|
|
|
|
// y = abs(x)
|
|
// y >>= 12 // Shift by 2*ϕ - 8.
|
|
// y = min(y, fxAlmost256)
|
|
//
|
|
// pabsd %xmm1,%xmm2
|
|
// psrld $0xc,%xmm2
|
|
// pminud %xmm5,%xmm2
|
|
//
|
|
// Hopefully we'll get these opcode mnemonics into the assembler for Go
|
|
// 1.8. https://golang.org/issue/16007 isn't exactly the same thing, but
|
|
// it's similar.
|
|
BYTE $0x66; BYTE $0x0f; BYTE $0x38; BYTE $0x1e; BYTE $0xd1
|
|
BYTE $0x66; BYTE $0x0f; BYTE $0x72; BYTE $0xd2; BYTE $0x0c
|
|
BYTE $0x66; BYTE $0x0f; BYTE $0x38; BYTE $0x3b; BYTE $0xd5
|
|
|
|
// dst[0] = uint8(y)
|
|
MOVL X2, BX
|
|
MOVB BX, (DI)
|
|
|
|
// offset = x
|
|
MOVOU X1, X7
|
|
|
|
// i += 1
|
|
// dst = dst[1:]
|
|
// src = src[1:]
|
|
ADDQ $1, AX
|
|
ADDQ $1, DI
|
|
ADDQ $4, SI
|
|
JMP fxAccOpSrcLoop1
|
|
|
|
fxAccOpSrcEnd:
|
|
RET
|
|
|
|
// ----------------------------------------------------------------------------
|
|
|
|
// func floatingAccumulateOpSrcSIMD(dst []uint8, src []float32)
|
|
//
|
|
// XMM registers. Variable names are per
|
|
// https://github.com/google/font-rs/blob/master/src/accumulate.c
|
|
//
|
|
// xmm0 scratch
|
|
// xmm1 x
|
|
// xmm2 y, z
|
|
// xmm3 flAlmost256
|
|
// xmm4 flOnes
|
|
// xmm5 flSignMask
|
|
// xmm6 shuffleMask
|
|
// xmm7 offset
|
|
TEXT ·floatingAccumulateOpSrcSIMD(SB), NOSPLIT, $8-48
|
|
MOVQ dst_base+0(FP), DI
|
|
MOVQ dst_len+8(FP), BX
|
|
MOVQ src_base+24(FP), SI
|
|
MOVQ src_len+32(FP), CX
|
|
|
|
// Sanity check that len(dst) >= len(src).
|
|
CMPQ BX, CX
|
|
JLT flAccOpSrcEnd
|
|
|
|
// CX = len(src) &^ 3
|
|
// DX = len(src)
|
|
MOVQ CX, DX
|
|
ANDQ $-4, CX
|
|
|
|
// Set MXCSR bits 13 and 14, so that the CVTPS2PL below is "Round To Zero".
|
|
STMXCSR mxcsrOrig-8(SP)
|
|
MOVL mxcsrOrig-8(SP), AX
|
|
ORL $0x6000, AX
|
|
MOVL AX, mxcsrNew-4(SP)
|
|
LDMXCSR mxcsrNew-4(SP)
|
|
|
|
// flAlmost256 := XMM(0x437fffff repeated four times) // 255.99998 as a float32.
|
|
// flOnes := XMM(0x3f800000 repeated four times) // 1 as a float32.
|
|
// flSignMask := XMM(0x7fffffff repeated four times) // All but the sign bit of a float32.
|
|
// shuffleMask := XMM(0x0c080400 repeated four times) // PSHUFB shuffle mask.
|
|
// offset := XMM(0x00000000 repeated four times) // Cumulative sum.
|
|
MOVOU flAlmost256<>(SB), X3
|
|
MOVOU flOnes<>(SB), X4
|
|
MOVOU flSignMask<>(SB), X5
|
|
MOVOU shuffleMask<>(SB), X6
|
|
XORPS X7, X7
|
|
|
|
// i := 0
|
|
MOVQ $0, AX
|
|
|
|
flAccOpSrcLoop4:
|
|
// for i < (len(src) &^ 3)
|
|
CMPQ AX, CX
|
|
JAE flAccOpSrcLoop1
|
|
|
|
// x = XMM(s0, s1, s2, s3)
|
|
//
|
|
// Where s0 is src[i+0], s1 is src[i+1], etc.
|
|
MOVOU (SI), X1
|
|
|
|
// scratch = XMM(0, s0, s1, s2)
|
|
// x += scratch // yields x == XMM(s0, s0+s1, s1+s2, s2+s3)
|
|
MOVOU X1, X0
|
|
PSLLO $4, X0
|
|
ADDPS X0, X1
|
|
|
|
// scratch = XMM(0, 0, 0, 0)
|
|
// scratch = XMM(scratch@0, scratch@0, x@0, x@1) // yields scratch == XMM(0, 0, s0, s0+s1)
|
|
// x += scratch // yields x == XMM(s0, s0+s1, s0+s1+s2, s0+s1+s2+s3)
|
|
XORPS X0, X0
|
|
SHUFPS $0x40, X1, X0
|
|
ADDPS X0, X1
|
|
|
|
// x += offset
|
|
ADDPS X7, X1
|
|
|
|
// y = x & flSignMask
|
|
// y = min(y, flOnes)
|
|
// y = mul(y, flAlmost256)
|
|
MOVOU X5, X2
|
|
ANDPS X1, X2
|
|
MINPS X4, X2
|
|
MULPS X3, X2
|
|
|
|
// z = float32ToInt32(y)
|
|
// z = shuffleTheLowBytesOfEach4ByteElement(z)
|
|
// copy(dst[:4], low4BytesOf(z))
|
|
CVTPS2PL X2, X2
|
|
PSHUFB X6, X2
|
|
MOVL X2, (DI)
|
|
|
|
// offset = XMM(x@3, x@3, x@3, x@3)
|
|
MOVOU X1, X7
|
|
SHUFPS $0xff, X1, X7
|
|
|
|
// i += 4
|
|
// dst = dst[4:]
|
|
// src = src[4:]
|
|
ADDQ $4, AX
|
|
ADDQ $4, DI
|
|
ADDQ $16, SI
|
|
JMP flAccOpSrcLoop4
|
|
|
|
flAccOpSrcLoop1:
|
|
// for i < len(src)
|
|
CMPQ AX, DX
|
|
JAE flAccOpSrcRestoreMXCSR
|
|
|
|
// x = src[i] + offset
|
|
MOVL (SI), X1
|
|
ADDPS X7, X1
|
|
|
|
// y = x & flSignMask
|
|
// y = min(y, flOnes)
|
|
// y = mul(y, flAlmost256)
|
|
MOVOU X5, X2
|
|
ANDPS X1, X2
|
|
MINPS X4, X2
|
|
MULPS X3, X2
|
|
|
|
// z = float32ToInt32(y)
|
|
// dst[0] = uint8(z)
|
|
CVTPS2PL X2, X2
|
|
MOVL X2, BX
|
|
MOVB BX, (DI)
|
|
|
|
// offset = x
|
|
MOVOU X1, X7
|
|
|
|
// i += 1
|
|
// dst = dst[1:]
|
|
// src = src[1:]
|
|
ADDQ $1, AX
|
|
ADDQ $1, DI
|
|
ADDQ $4, SI
|
|
JMP flAccOpSrcLoop1
|
|
|
|
flAccOpSrcRestoreMXCSR:
|
|
LDMXCSR mxcsrOrig-8(SP)
|
|
|
|
flAccOpSrcEnd:
|
|
RET
|