Skip to content

Commit

Permalink
cmd/compile: implement non-constant rotates
Browse files Browse the repository at this point in the history 
Makes math/bits.Rotate{Left,Right} fast on amd64.

name              old time/op  new time/op  delta
RotateLeft-12     7.42ns ± 6%  5.45ns ± 6%  -26.54%   (p=0.000 n=9+10)
RotateLeft8-12    4.77ns ± 5%  3.42ns ± 7%  -28.25%   (p=0.000 n=8+10)
RotateLeft16-12   4.82ns ± 8%  3.40ns ± 7%  -29.36%  (p=0.000 n=10+10)
RotateLeft32-12   4.87ns ± 7%  3.48ns ± 7%  -28.51%    (p=0.000 n=8+9)
RotateLeft64-12   5.23ns ±10%  3.35ns ± 6%  -35.97%   (p=0.000 n=9+10)
RotateRight-12    7.59ns ± 8%  5.71ns ± 1%  -24.72%   (p=0.000 n=10+8)
RotateRight8-12   4.98ns ± 7%  3.36ns ± 9%  -32.55%  (p=0.000 n=10+10)
RotateRight16-12  5.12ns ± 2%  3.45ns ± 5%  -32.62%  (p=0.000 n=10+10)
RotateRight32-12  4.80ns ± 6%  3.42ns ±16%  -28.68%  (p=0.000 n=10+10)
RotateRight64-12  4.78ns ± 6%  3.42ns ± 6%  -28.50%  (p=0.000 n=10+10)

Update #18940

Change-Id: Ie79fb5581c489ed4d3b859314c5e669a134c119b
Reviewed-on: https://go-review.googlesource.com/39711
Run-TryBot: Keith Randall <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Josh Bleecher Snyder <[email protected]>
  • Loading branch information
@randall77
randall77 committed Apr 17, 2017
1 parent 3852100 commit 7e07e63
Show file tree
Hide file tree
Showing 6 changed files with 13,619 additions and 7,823 deletions.
2 changes: 2 additions & 0 deletions src/cmd/compile/internal/amd64/ssa.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,8 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
ssa.OpAMD64SHLQ, ssa.OpAMD64SHLL,
ssa.OpAMD64SHRQ, ssa.OpAMD64SHRL, ssa.OpAMD64SHRW, ssa.OpAMD64SHRB,
ssa.OpAMD64SARQ, ssa.OpAMD64SARL, ssa.OpAMD64SARW, ssa.OpAMD64SARB,
ssa.OpAMD64ROLQ, ssa.OpAMD64ROLL, ssa.OpAMD64ROLW, ssa.OpAMD64ROLB,
ssa.OpAMD64RORQ, ssa.OpAMD64RORL, ssa.OpAMD64RORW, ssa.OpAMD64RORB,
ssa.OpAMD64ADDSS, ssa.OpAMD64ADDSD, ssa.OpAMD64SUBSS, ssa.OpAMD64SUBSD,
ssa.OpAMD64MULSS, ssa.OpAMD64MULSD, ssa.OpAMD64DIVSS, ssa.OpAMD64DIVSD,
ssa.OpAMD64PXOR:
Expand Down
57 changes: 57 additions & 0 deletions src/cmd/compile/internal/gc/asm_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -800,6 +800,63 @@ var linuxAMD64Tests = []*asmTest{
}`,
[]string{"\tCMPQ\t[A-Z]"},
},
// Non-constant rotate
{
`func rot64l(x uint64, y int) uint64 {
z := uint(y & 63)
return x << z | x >> (64-z)
}`,
[]string{"\tROLQ\t"},
},
{
`func rot64r(x uint64, y int) uint64 {
z := uint(y & 63)
return x >> z | x << (64-z)
}`,
[]string{"\tRORQ\t"},
},
{
`func rot32l(x uint32, y int) uint32 {
z := uint(y & 31)
return x << z | x >> (32-z)
}`,
[]string{"\tROLL\t"},
},
{
`func rot32r(x uint32, y int) uint32 {
z := uint(y & 31)
return x >> z | x << (32-z)
}`,
[]string{"\tRORL\t"},
},
{
`func rot16l(x uint16, y int) uint16 {
z := uint(y & 15)
return x << z | x >> (16-z)
}`,
[]string{"\tROLW\t"},
},
{
`func rot16r(x uint16, y int) uint16 {
z := uint(y & 15)
return x >> z | x << (16-z)
}`,
[]string{"\tRORW\t"},
},
{
`func rot8l(x uint8, y int) uint8 {
z := uint(y & 7)
return x << z | x >> (8-z)
}`,
[]string{"\tROLB\t"},
},
{
`func rot8r(x uint8, y int) uint8 {
z := uint(y & 7)
return x >> z | x << (8-z)
}`,
[]string{"\tRORB\t"},
},
}

var linux386Tests = []*asmTest{
Expand Down
171 changes: 156 additions & 15 deletions src/cmd/compile/internal/ssa/gen/AMD64.rules
View file
Original file line number Diff line number Diff line change
Expand Up @@ -621,19 +621,60 @@
(SARB x (MOVQconst [c])) -> (SARBconst [min(c&31,7)] x)
(SARB x (MOVLconst [c])) -> (SARBconst [min(c&31,7)] x)

(SARL x (ANDLconst [31] y)) -> (SARL x y)
(SARQ x (ANDQconst [63] y)) -> (SARQ x y)

(SHLL x (ANDLconst [31] y)) -> (SHLL x y)
(SHLQ x (ANDQconst [63] y)) -> (SHLQ x y)
(SHLQ x (ANDLconst [63] y)) -> (SHLQ x y)

(SHRL x (ANDLconst [31] y)) -> (SHRL x y)
(SHRQ x (ANDQconst [63] y)) -> (SHRQ x y)
(SHRQ x (ANDLconst [63] y)) -> (SHRQ x y)

// Rotate instructions

// Operations which don't affect the low 6/5 bits of the shift amount are NOPs.
(SHLQ x (ADDQconst [c] y)) && c & 63 == 0 -> (SHLQ x y)
(SHRQ x (ADDQconst [c] y)) && c & 63 == 0 -> (SHRQ x y)
(SARQ x (ADDQconst [c] y)) && c & 63 == 0 -> (SARQ x y)
(SHLQ x (NEGQ <t> (ADDQconst [c] y))) && c & 63 == 0 -> (SHLQ x (NEGQ <t> y))
(SHRQ x (NEGQ <t> (ADDQconst [c] y))) && c & 63 == 0 -> (SHRQ x (NEGQ <t> y))
(SARQ x (NEGQ <t> (ADDQconst [c] y))) && c & 63 == 0 -> (SARQ x (NEGQ <t> y))
(SHLQ x (ANDQconst [c] y)) && c & 63 == 63 -> (SHLQ x y)
(SHRQ x (ANDQconst [c] y)) && c & 63 == 63 -> (SHRQ x y)
(SARQ x (ANDQconst [c] y)) && c & 63 == 63 -> (SARQ x y)
(SHLQ x (NEGQ <t> (ANDQconst [c] y))) && c & 63 == 63 -> (SHLQ x (NEGQ <t> y))
(SHRQ x (NEGQ <t> (ANDQconst [c] y))) && c & 63 == 63 -> (SHRQ x (NEGQ <t> y))
(SARQ x (NEGQ <t> (ANDQconst [c] y))) && c & 63 == 63 -> (SARQ x (NEGQ <t> y))

(SHLL x (ADDQconst [c] y)) && c & 31 == 0 -> (SHLL x y)
(SHRL x (ADDQconst [c] y)) && c & 31 == 0 -> (SHRL x y)
(SARL x (ADDQconst [c] y)) && c & 31 == 0 -> (SARL x y)
(SHLL x (NEGQ <t> (ADDQconst [c] y))) && c & 31 == 0 -> (SHLL x (NEGQ <t> y))
(SHRL x (NEGQ <t> (ADDQconst [c] y))) && c & 31 == 0 -> (SHRL x (NEGQ <t> y))
(SARL x (NEGQ <t> (ADDQconst [c] y))) && c & 31 == 0 -> (SARL x (NEGQ <t> y))
(SHLL x (ANDQconst [c] y)) && c & 31 == 31 -> (SHLL x y)
(SHRL x (ANDQconst [c] y)) && c & 31 == 31 -> (SHRL x y)
(SARL x (ANDQconst [c] y)) && c & 31 == 31 -> (SARL x y)
(SHLL x (NEGQ <t> (ANDQconst [c] y))) && c & 31 == 31 -> (SHLL x (NEGQ <t> y))
(SHRL x (NEGQ <t> (ANDQconst [c] y))) && c & 31 == 31 -> (SHRL x (NEGQ <t> y))
(SARL x (NEGQ <t> (ANDQconst [c] y))) && c & 31 == 31 -> (SARL x (NEGQ <t> y))

(SHLQ x (ADDLconst [c] y)) && c & 63 == 0 -> (SHLQ x y)
(SHRQ x (ADDLconst [c] y)) && c & 63 == 0 -> (SHRQ x y)
(SARQ x (ADDLconst [c] y)) && c & 63 == 0 -> (SARQ x y)
(SHLQ x (NEGL <t> (ADDLconst [c] y))) && c & 63 == 0 -> (SHLQ x (NEGL <t> y))
(SHRQ x (NEGL <t> (ADDLconst [c] y))) && c & 63 == 0 -> (SHRQ x (NEGL <t> y))
(SARQ x (NEGL <t> (ADDLconst [c] y))) && c & 63 == 0 -> (SARQ x (NEGL <t> y))
(SHLQ x (ANDLconst [c] y)) && c & 63 == 63 -> (SHLQ x y)
(SHRQ x (ANDLconst [c] y)) && c & 63 == 63 -> (SHRQ x y)
(SARQ x (ANDLconst [c] y)) && c & 63 == 63 -> (SARQ x y)
(SHLQ x (NEGL <t> (ANDLconst [c] y))) && c & 63 == 63 -> (SHLQ x (NEGL <t> y))
(SHRQ x (NEGL <t> (ANDLconst [c] y))) && c & 63 == 63 -> (SHRQ x (NEGL <t> y))
(SARQ x (NEGL <t> (ANDLconst [c] y))) && c & 63 == 63 -> (SARQ x (NEGL <t> y))

(SHLL x (ADDLconst [c] y)) && c & 31 == 0 -> (SHLL x y)
(SHRL x (ADDLconst [c] y)) && c & 31 == 0 -> (SHRL x y)
(SARL x (ADDLconst [c] y)) && c & 31 == 0 -> (SARL x y)
(SHLL x (NEGL <t> (ADDLconst [c] y))) && c & 31 == 0 -> (SHLL x (NEGL <t> y))
(SHRL x (NEGL <t> (ADDLconst [c] y))) && c & 31 == 0 -> (SHRL x (NEGL <t> y))
(SARL x (NEGL <t> (ADDLconst [c] y))) && c & 31 == 0 -> (SARL x (NEGL <t> y))
(SHLL x (ANDLconst [c] y)) && c & 31 == 31 -> (SHLL x y)
(SHRL x (ANDLconst [c] y)) && c & 31 == 31 -> (SHRL x y)
(SARL x (ANDLconst [c] y)) && c & 31 == 31 -> (SARL x y)
(SHLL x (NEGL <t> (ANDLconst [c] y))) && c & 31 == 31 -> (SHLL x (NEGL <t> y))
(SHRL x (NEGL <t> (ANDLconst [c] y))) && c & 31 == 31 -> (SHRL x (NEGL <t> y))
(SARL x (NEGL <t> (ANDLconst [c] y))) && c & 31 == 31 -> (SARL x (NEGL <t> y))

// Constant rotate instructions
(ADDQ (SHLQconst x [c]) (SHRQconst x [d])) && d==64-c -> (ROLQconst x [c])
( ORQ (SHLQconst x [c]) (SHRQconst x [d])) && d==64-c -> (ROLQconst x [c])
(XORQ (SHLQconst x [c]) (SHRQconst x [d])) && d==64-c -> (ROLQconst x [c])
Expand All @@ -655,10 +696,109 @@
(ROLWconst [c] (ROLWconst [d] x)) -> (ROLWconst [(c+d)&15] x)
(ROLBconst [c] (ROLBconst [d] x)) -> (ROLBconst [(c+d)& 7] x)

// TODO: non-constant rotates if shift amount is known to be bounded (shift & 63 or something).
// Non-constant rotates.
// We want to issue a rotate when the Go source contains code like
// y &= 63
// x << y | x >> (64-y)
// The shift rules above convert << to SHLx and >> to SHRx.
// SHRx converts its shift argument from 64-y to -y.
// A tricky situation occurs when y==0. Then the original code would be:
// x << 0 | x >> 64
// But x >> 64 is 0, not x. So there's an additional mask that is ANDed in
// to force the second term to 0. We don't need that mask, but we must match
// it in order to strip it out.
(ORQ (SHLQ x y) (ANDQ (SHRQ x (NEGQ y)) (SBBQcarrymask (CMPQconst (NEGQ (ADDQconst (ANDQconst y [63]) [-64])) [64])))) -> (ROLQ x y)
(ORQ (SHLQ x y) (ANDQ (SHRQ x (NEGL y)) (SBBQcarrymask (CMPLconst (NEGL (ADDLconst (ANDLconst y [63]) [-64])) [64])))) -> (ROLQ x y)
(ORQ (SHRQ x y) (ANDQ (SHLQ x (NEGQ y)) (SBBQcarrymask (CMPQconst (NEGQ (ADDQconst (ANDQconst y [63]) [-64])) [64])))) -> (RORQ x y)
(ORQ (SHRQ x y) (ANDQ (SHLQ x (NEGL y)) (SBBQcarrymask (CMPLconst (NEGL (ADDLconst (ANDLconst y [63]) [-64])) [64])))) -> (RORQ x y)

(ORL (SHLL x y) (ANDL (SHRL x (NEGQ y)) (SBBLcarrymask (CMPQconst (NEGQ (ADDQconst (ANDQconst y [31]) [-32])) [32])))) -> (ROLL x y)
(ORL (SHLL x y) (ANDL (SHRL x (NEGL y)) (SBBLcarrymask (CMPLconst (NEGL (ADDLconst (ANDLconst y [31]) [-32])) [32])))) -> (ROLL x y)
(ORL (SHRL x y) (ANDL (SHLL x (NEGQ y)) (SBBLcarrymask (CMPQconst (NEGQ (ADDQconst (ANDQconst y [31]) [-32])) [32])))) -> (RORL x y)
(ORL (SHRL x y) (ANDL (SHLL x (NEGL y)) (SBBLcarrymask (CMPLconst (NEGL (ADDLconst (ANDLconst y [31]) [-32])) [32])))) -> (RORL x y)

// Help with rotate detection
(CMPQconst (NEGQ (ADDQconst [-16] (ANDQconst [15] _))) [32]) -> (FlagLT_ULT)
(CMPQconst (NEGQ (ADDQconst [ -8] (ANDQconst [7] _))) [32]) -> (FlagLT_ULT)

(ORL (SHLL x (ANDQconst y [15]))
(ANDL (SHRW x (NEGQ (ADDQconst (ANDQconst y [15]) [-16])))
(SBBLcarrymask (CMPQconst (NEGQ (ADDQconst (ANDQconst y [15]) [-16])) [16]))))
&& v.Type.Size() == 2
-> (ROLW x y)
(ORL (SHLL x (ANDLconst y [15]))
(ANDL (SHRW x (NEGL (ADDLconst (ANDLconst y [15]) [-16])))
(SBBLcarrymask (CMPLconst (NEGL (ADDLconst (ANDLconst y [15]) [-16])) [16]))))
&& v.Type.Size() == 2
-> (ROLW x y)
(ORL (SHRW x (ANDQconst y [15]))
(SHLL x (NEGQ (ADDQconst (ANDQconst y [15]) [-16]))))
&& v.Type.Size() == 2
-> (RORW x y)
(ORL (SHRW x (ANDLconst y [15]))
(SHLL x (NEGL (ADDLconst (ANDLconst y [15]) [-16]))))
&& v.Type.Size() == 2
-> (RORW x y)

(ORL (SHLL x (ANDQconst y [ 7]))
(ANDL (SHRB x (NEGQ (ADDQconst (ANDQconst y [ 7]) [ -8])))
(SBBLcarrymask (CMPQconst (NEGQ (ADDQconst (ANDQconst y [ 7]) [ -8])) [ 8]))))
&& v.Type.Size() == 1
-> (ROLB x y)
(ORL (SHLL x (ANDLconst y [ 7]))
(ANDL (SHRB x (NEGL (ADDLconst (ANDLconst y [ 7]) [ -8])))
(SBBLcarrymask (CMPLconst (NEGL (ADDLconst (ANDLconst y [ 7]) [ -8])) [ 8]))))
&& v.Type.Size() == 1
-> (ROLB x y)
(ORL (SHRB x (ANDQconst y [ 7]))
(SHLL x (NEGQ (ADDQconst (ANDQconst y [ 7]) [ -8]))))
&& v.Type.Size() == 1
-> (RORB x y)
(ORL (SHRB x (ANDLconst y [ 7]))
(SHLL x (NEGL (ADDLconst (ANDLconst y [ 7]) [ -8]))))
&& v.Type.Size() == 1
-> (RORB x y)

// rotate left negative = rotate right
(ROLQ x (NEGQ y)) -> (RORQ x y)
(ROLQ x (NEGL y)) -> (RORQ x y)
(ROLL x (NEGQ y)) -> (RORL x y)
(ROLL x (NEGL y)) -> (RORL x y)
(ROLW x (NEGQ y)) -> (RORW x y)
(ROLW x (NEGL y)) -> (RORW x y)
(ROLB x (NEGQ y)) -> (RORB x y)
(ROLB x (NEGL y)) -> (RORB x y)

// rotate right negative = rotate left
(RORQ x (NEGQ y)) -> (ROLQ x y)
(RORQ x (NEGL y)) -> (ROLQ x y)
(RORL x (NEGQ y)) -> (ROLL x y)
(RORL x (NEGL y)) -> (ROLL x y)
(RORW x (NEGQ y)) -> (ROLW x y)
(RORW x (NEGL y)) -> (ROLW x y)
(RORB x (NEGQ y)) -> (ROLB x y)
(RORB x (NEGL y)) -> (ROLB x y)

// rotate by constants
(ROLQ x (MOVQconst [c])) -> (ROLQconst [c&63] x)
(ROLQ x (MOVLconst [c])) -> (ROLQconst [c&63] x)
(ROLL x (MOVQconst [c])) -> (ROLLconst [c&31] x)
(ROLL x (MOVLconst [c])) -> (ROLLconst [c&31] x)
(ROLW x (MOVQconst [c])) -> (ROLWconst [c&15] x)
(ROLW x (MOVLconst [c])) -> (ROLWconst [c&15] x)
(ROLB x (MOVQconst [c])) -> (ROLBconst [c&7 ] x)
(ROLB x (MOVLconst [c])) -> (ROLBconst [c&7 ] x)

(RORQ x (MOVQconst [c])) -> (ROLQconst [(-c)&63] x)
(RORQ x (MOVLconst [c])) -> (ROLQconst [(-c)&63] x)
(RORL x (MOVQconst [c])) -> (ROLLconst [(-c)&31] x)
(RORL x (MOVLconst [c])) -> (ROLLconst [(-c)&31] x)
(RORW x (MOVQconst [c])) -> (ROLWconst [(-c)&15] x)
(RORW x (MOVLconst [c])) -> (ROLWconst [(-c)&15] x)
(RORB x (MOVQconst [c])) -> (ROLBconst [(-c)&7 ] x)
(RORB x (MOVLconst [c])) -> (ROLBconst [(-c)&7 ] x)

// Constant shift simplifications

(SHLQconst x [0]) -> x
(SHRQconst x [0]) -> x
(SARQconst x [0]) -> x
Expand Down Expand Up @@ -1171,6 +1311,7 @@
(CMPLconst (ANDLconst _ [m]) [n]) && 0 <= int32(m) && int32(m) < int32(n) -> (FlagLT_ULT)
(CMPWconst (ANDLconst _ [m]) [n]) && 0 <= int16(m) && int16(m) < int16(n) -> (FlagLT_ULT)
(CMPBconst (ANDLconst _ [m]) [n]) && 0 <= int8(m) && int8(m) < int8(n) -> (FlagLT_ULT)

// TODO: DIVxU also.

// Absorb flag constants into SBB ops.
Expand Down
8 changes: 8 additions & 0 deletions src/cmd/compile/internal/ssa/gen/AMD64Ops.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,14 @@ func init() {
{name: "SARWconst", argLength: 1, reg: gp11, asm: "SARW", aux: "Int8", resultInArg0: true, clobberFlags: true}, // signed arg0 >> auxint, shift amount 0-15
{name: "SARBconst", argLength: 1, reg: gp11, asm: "SARB", aux: "Int8", resultInArg0: true, clobberFlags: true}, // signed arg0 >> auxint, shift amount 0-7

{name: "ROLQ", argLength: 2, reg: gp21shift, asm: "ROLQ", resultInArg0: true, clobberFlags: true}, // arg0 rotate left arg1 bits.
{name: "ROLL", argLength: 2, reg: gp21shift, asm: "ROLL", resultInArg0: true, clobberFlags: true}, // arg0 rotate left arg1 bits.
{name: "ROLW", argLength: 2, reg: gp21shift, asm: "ROLW", resultInArg0: true, clobberFlags: true}, // arg0 rotate left arg1 bits.
{name: "ROLB", argLength: 2, reg: gp21shift, asm: "ROLB", resultInArg0: true, clobberFlags: true}, // arg0 rotate left arg1 bits.
{name: "RORQ", argLength: 2, reg: gp21shift, asm: "RORQ", resultInArg0: true, clobberFlags: true}, // arg0 rotate right arg1 bits.
{name: "RORL", argLength: 2, reg: gp21shift, asm: "RORL", resultInArg0: true, clobberFlags: true}, // arg0 rotate right arg1 bits.
{name: "RORW", argLength: 2, reg: gp21shift, asm: "RORW", resultInArg0: true, clobberFlags: true}, // arg0 rotate right arg1 bits.
{name: "RORB", argLength: 2, reg: gp21shift, asm: "RORB", resultInArg0: true, clobberFlags: true}, // arg0 rotate right arg1 bits.
{name: "ROLQconst", argLength: 1, reg: gp11, asm: "ROLQ", aux: "Int8", resultInArg0: true, clobberFlags: true}, // arg0 rotate left auxint, rotate amount 0-63
{name: "ROLLconst", argLength: 1, reg: gp11, asm: "ROLL", aux: "Int8", resultInArg0: true, clobberFlags: true}, // arg0 rotate left auxint, rotate amount 0-31
{name: "ROLWconst", argLength: 1, reg: gp11, asm: "ROLW", aux: "Int8", resultInArg0: true, clobberFlags: true}, // arg0 rotate left auxint, rotate amount 0-15
Expand Down
Loading

0 comments on commit 7e07e63

Please sign in to comment.