diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll index 3b9c96dc50f06df..fe649d433304178 100644 --- a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll +++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll @@ -1,71 +1,140 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10 %s +; RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX11 %s define amdgpu_ps float @_amdgpu_ps_main() #0 { -; GCN-LABEL: _amdgpu_ps_main: -; GCN: ; %bb.0: ; %.entry -; GCN-NEXT: image_sample v[0:1], v[0:1], s[0:7], s[0:3] dmask:0x3 dim:SQ_RSRC_IMG_2D -; GCN-NEXT: v_mov_b32_e32 v4, 0 -; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: s_clause 0x1 -; GCN-NEXT: image_sample v2, v[0:1], s[0:7], s[0:3] dmask:0x4 dim:SQ_RSRC_IMG_2D -; GCN-NEXT: image_sample v3, v[0:1], s[0:7], s[0:3] dmask:0x1 dim:SQ_RSRC_IMG_2D -; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: image_load_mip v4, v[2:4], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D unorm -; GCN-NEXT: s_clause 0x3 -; GCN-NEXT: s_buffer_load_dword s24, s[0:3], 0x5c -; GCN-NEXT: s_buffer_load_dword s28, s[0:3], 0x7c -; GCN-NEXT: s_buffer_load_dword s29, s[0:3], 0xc0 -; GCN-NEXT: s_waitcnt_depctr 0xffe3 -; GCN-NEXT: s_nop 0 -; GCN-NEXT: s_buffer_load_dwordx4 s[0:3], s[0:3], 0x40 -; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_clause 0x1 -; GCN-NEXT: s_buffer_load_dwordx4 s[4:7], s[0:3], 0x50 -; GCN-NEXT: s_nop 0 -; GCN-NEXT: s_buffer_load_dword s0, s[0:3], 0x2c -; GCN-NEXT: v_sub_f32_e64 v5, s24, s28 -; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_clause 0x4 -; GCN-NEXT: s_buffer_load_dwordx4 s[8:11], s[0:3], 0x60 -; GCN-NEXT: s_buffer_load_dwordx4 s[12:15], s[0:3], 0x20 -; GCN-NEXT: s_buffer_load_dwordx4 s[16:19], s[0:3], 0x0 -; GCN-NEXT: s_buffer_load_dwordx4 s[20:23], s[0:3], 0x70 -; GCN-NEXT: s_buffer_load_dwordx4 s[24:27], s[0:3], 0x10 -; GCN-NEXT: v_fma_f32 v1, v1, v5, s28 -; GCN-NEXT: v_max_f32_e64 v6, s0, s0 clamp -; GCN-NEXT: v_add_f32_e64 v5, s29, -1.0 -; GCN-NEXT: v_sub_f32_e32 v8, s0, v1 -; GCN-NEXT: v_fma_f32 v7, -s2, v6, s6 -; GCN-NEXT: v_fma_f32 v5, v6, v5, 1.0 -; GCN-NEXT: v_mad_f32 v10, s2, v6, v2 -; GCN-NEXT: s_mov_b32 s0, 0x3c23d70a -; GCN-NEXT: v_fmac_f32_e32 v1, v6, v8 -; GCN-NEXT: v_fmac_f32_e32 v10, v7, v6 -; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: v_mul_f32_e32 v9, s10, v0 -; GCN-NEXT: v_fma_f32 v0, -v0, s10, s14 -; GCN-NEXT: v_mul_f32_e32 v8, s18, v2 -; GCN-NEXT: v_mul_f32_e32 v3, s22, v3 -; GCN-NEXT: v_fmac_f32_e32 v9, v0, v6 -; GCN-NEXT: v_sub_f32_e32 v0, v1, v5 -; GCN-NEXT: v_mul_f32_e32 v1, v8, v6 -; GCN-NEXT: v_mul_f32_e32 v7, v6, v3 -; GCN-NEXT: v_fma_f32 v3, -v6, v3, v9 -; GCN-NEXT: v_fmac_f32_e32 v5, v0, v6 -; GCN-NEXT: v_fma_f32 v0, v2, s26, -v1 -; GCN-NEXT: v_fmac_f32_e32 v7, v3, v6 -; GCN-NEXT: v_fmac_f32_e32 v1, v0, v6 -; GCN-NEXT: v_mul_f32_e32 v0, v2, v6 -; GCN-NEXT: s_waitcnt vmcnt(0) -; GCN-NEXT: v_add_f32_e32 v4, v4, v10 -; GCN-NEXT: v_mul_f32_e32 v3, v4, v6 -; GCN-NEXT: v_fmaak_f32 v4, s0, v5, 0x3ca3d70a -; GCN-NEXT: v_mul_f32_e32 v1, v3, v1 -; GCN-NEXT: v_mul_f32_e32 v2, v7, v4 -; GCN-NEXT: v_fmac_f32_e32 v1, v2, v0 -; GCN-NEXT: v_max_f32_e32 v0, 0, v1 -; GCN-NEXT: ; return to shader part epilog +; GFX10-LABEL: _amdgpu_ps_main: +; GFX10: ; %bb.0: ; %.entry +; GFX10-NEXT: image_sample v[0:1], v[0:1], s[0:7], s[0:3] dmask:0x3 dim:SQ_RSRC_IMG_2D +; GFX10-NEXT: v_mov_b32_e32 v4, 0 +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: s_clause 0x1 +; GFX10-NEXT: image_sample v2, v[0:1], s[0:7], s[0:3] dmask:0x4 dim:SQ_RSRC_IMG_2D +; GFX10-NEXT: image_sample v3, v[0:1], s[0:7], s[0:3] dmask:0x1 dim:SQ_RSRC_IMG_2D +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: image_load_mip v4, v[2:4], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D unorm +; GFX10-NEXT: s_clause 0x3 +; GFX10-NEXT: s_buffer_load_dword s24, s[0:3], 0x5c +; GFX10-NEXT: s_buffer_load_dword s28, s[0:3], 0x7c +; GFX10-NEXT: s_buffer_load_dword s29, s[0:3], 0xc0 +; GFX10-NEXT: s_waitcnt_depctr 0xffe3 +; GFX10-NEXT: s_nop 0 +; GFX10-NEXT: s_buffer_load_dwordx4 s[0:3], s[0:3], 0x40 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_clause 0x1 +; GFX10-NEXT: s_buffer_load_dwordx4 s[4:7], s[0:3], 0x50 +; GFX10-NEXT: s_nop 0 +; GFX10-NEXT: s_buffer_load_dword s0, s[0:3], 0x2c +; GFX10-NEXT: v_sub_f32_e64 v5, s24, s28 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_clause 0x4 +; GFX10-NEXT: s_buffer_load_dwordx4 s[8:11], s[0:3], 0x60 +; GFX10-NEXT: s_buffer_load_dwordx4 s[12:15], s[0:3], 0x20 +; GFX10-NEXT: s_buffer_load_dwordx4 s[16:19], s[0:3], 0x0 +; GFX10-NEXT: s_buffer_load_dwordx4 s[20:23], s[0:3], 0x70 +; GFX10-NEXT: s_buffer_load_dwordx4 s[24:27], s[0:3], 0x10 +; GFX10-NEXT: v_fma_f32 v1, v1, v5, s28 +; GFX10-NEXT: v_max_f32_e64 v6, s0, s0 clamp +; GFX10-NEXT: v_add_f32_e64 v5, s29, -1.0 +; GFX10-NEXT: v_sub_f32_e32 v8, s0, v1 +; GFX10-NEXT: v_fma_f32 v7, -s2, v6, s6 +; GFX10-NEXT: v_fma_f32 v5, v6, v5, 1.0 +; GFX10-NEXT: v_mad_f32 v10, s2, v6, v2 +; GFX10-NEXT: s_mov_b32 s0, 0x3c23d70a +; GFX10-NEXT: v_fmac_f32_e32 v1, v6, v8 +; GFX10-NEXT: v_fmac_f32_e32 v10, v7, v6 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: v_mul_f32_e32 v9, s10, v0 +; GFX10-NEXT: v_fma_f32 v0, -v0, s10, s14 +; GFX10-NEXT: v_mul_f32_e32 v8, s18, v2 +; GFX10-NEXT: v_mul_f32_e32 v3, s22, v3 +; GFX10-NEXT: v_fmac_f32_e32 v9, v0, v6 +; GFX10-NEXT: v_sub_f32_e32 v0, v1, v5 +; GFX10-NEXT: v_mul_f32_e32 v1, v8, v6 +; GFX10-NEXT: v_mul_f32_e32 v7, v6, v3 +; GFX10-NEXT: v_fma_f32 v3, -v6, v3, v9 +; GFX10-NEXT: v_fmac_f32_e32 v5, v0, v6 +; GFX10-NEXT: v_fma_f32 v0, v2, s26, -v1 +; GFX10-NEXT: v_fmac_f32_e32 v7, v3, v6 +; GFX10-NEXT: v_fmac_f32_e32 v1, v0, v6 +; GFX10-NEXT: v_mul_f32_e32 v0, v2, v6 +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: v_add_f32_e32 v4, v4, v10 +; GFX10-NEXT: v_mul_f32_e32 v3, v4, v6 +; GFX10-NEXT: v_fmaak_f32 v4, s0, v5, 0x3ca3d70a +; GFX10-NEXT: v_mul_f32_e32 v1, v3, v1 +; GFX10-NEXT: v_mul_f32_e32 v2, v7, v4 +; GFX10-NEXT: v_fmac_f32_e32 v1, v2, v0 +; GFX10-NEXT: v_max_f32_e32 v0, 0, v1 +; GFX10-NEXT: ; return to shader part epilog +; +; GFX11-LABEL: _amdgpu_ps_main: +; GFX11: ; %bb.0: ; %.entry +; GFX11-NEXT: image_sample v[0:1], v[0:1], s[0:7], s[0:3] dmask:0x3 dim:SQ_RSRC_IMG_2D +; GFX11-NEXT: v_mov_b32_e32 v4, 0 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: s_clause 0x1 +; GFX11-NEXT: image_sample v2, v[0:1], s[0:7], s[0:3] dmask:0x4 dim:SQ_RSRC_IMG_2D +; GFX11-NEXT: image_sample v3, v[0:1], s[0:7], s[0:3] dmask:0x1 dim:SQ_RSRC_IMG_2D +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: image_load_mip v4, v[2:4], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D unorm +; GFX11-NEXT: s_clause 0x3 +; GFX11-NEXT: s_buffer_load_b32 s24, s[0:3], 0x5c +; GFX11-NEXT: s_buffer_load_b32 s28, s[0:3], 0x7c +; GFX11-NEXT: s_buffer_load_b32 s29, s[0:3], 0xc0 +; GFX11-NEXT: s_buffer_load_b128 s[0:3], s[0:3], 0x40 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_clause 0x1 +; GFX11-NEXT: s_buffer_load_b128 s[4:7], s[0:3], 0x50 +; GFX11-NEXT: s_buffer_load_b32 s0, s[0:3], 0x2c +; GFX11-NEXT: v_sub_f32_e64 v5, s24, s28 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_clause 0x3 +; GFX11-NEXT: s_buffer_load_b128 s[8:11], s[0:3], 0x60 +; GFX11-NEXT: s_buffer_load_b128 s[12:15], s[0:3], 0x20 +; GFX11-NEXT: s_buffer_load_b128 s[16:19], s[0:3], 0x0 +; GFX11-NEXT: s_buffer_load_b128 s[20:23], s[0:3], 0x70 +; GFX11-NEXT: v_fma_f32 v1, v1, v5, s28 +; GFX11-NEXT: v_max_f32_e64 v6, s0, s0 clamp +; GFX11-NEXT: s_buffer_load_b128 s[24:27], s[0:3], 0x10 +; GFX11-NEXT: v_add_f32_e64 v5, s29, -1.0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_sub_f32_e32 v8, s0, v1 +; GFX11-NEXT: v_fma_f32 v7, -s2, v6, s6 +; GFX11-NEXT: v_fma_f32 v10, s2, v6, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-NEXT: v_fma_f32 v5, v6, v5, 1.0 +; GFX11-NEXT: s_mov_b32 s0, 0x3c23d70a +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: v_mul_f32_e32 v9, s10, v0 +; GFX11-NEXT: v_fma_f32 v0, -v0, s10, s14 +; GFX11-NEXT: v_mul_f32_e32 v3, s22, v3 +; GFX11-NEXT: v_dual_fmac_f32 v1, v6, v8 :: v_dual_mul_f32 v8, s18, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_fmac_f32_e32 v9, v0, v6 +; GFX11-NEXT: v_dual_fmac_f32 v10, v7, v6 :: v_dual_mul_f32 v7, v6, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_sub_f32_e32 v0, v1, v5 +; GFX11-NEXT: v_fma_f32 v3, -v6, v3, v9 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_fmac_f32_e32 v7, v3, v6 +; GFX11-NEXT: v_fmac_f32_e32 v5, v0, v6 +; GFX11-NEXT: v_mul_f32_e32 v1, v8, v6 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_add_f32_e32 v4, v4, v10 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_dual_mul_f32 v3, v4, v6 :: v_dual_fmaak_f32 v4, s0, v5, 0x3ca3d70a +; GFX11-NEXT: v_fma_f32 v0, v2, s26, -v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_fmac_f32_e32 v1, v0, v6 +; GFX11-NEXT: v_mul_f32_e32 v0, v2, v6 +; GFX11-NEXT: v_mul_f32_e32 v2, v7, v4 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mul_f32_e32 v1, v3, v1 +; GFX11-NEXT: v_fmac_f32_e32 v1, v2, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_max_f32_e32 v0, 0, v1 +; GFX11-NEXT: ; return to shader part epilog .entry: %0 = call <3 x float> @llvm.amdgcn.image.sample.2d.v3f32.f32(i32 7, float undef, float undef, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) %.i2243 = extractelement <3 x float> %0, i32 2 @@ -168,13 +237,22 @@ define amdgpu_ps float @_amdgpu_ps_main() #0 { } define float @fmac_sequence_simple(float %a, float %b, float %c, float %d, float %e) #0 { -; GCN-LABEL: fmac_sequence_simple: -; GCN: ; %bb.0: -; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: v_fma_f32 v2, v2, v3, v4 -; GCN-NEXT: v_fmac_f32_e32 v2, v0, v1 -; GCN-NEXT: v_mov_b32_e32 v0, v2 -; GCN-NEXT: s_setpc_b64 s[30:31] +; GFX10-LABEL: fmac_sequence_simple: +; GFX10: ; %bb.0: +; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: v_fma_f32 v2, v2, v3, v4 +; GFX10-NEXT: v_fmac_f32_e32 v2, v0, v1 +; GFX10-NEXT: v_mov_b32_e32 v0, v2 +; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: fmac_sequence_simple: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_fma_f32 v2, v2, v3, v4 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_fmac_f32_e32 v2, v0, v1 +; GFX11-NEXT: v_mov_b32_e32 v0, v2 +; GFX11-NEXT: s_setpc_b64 s[30:31] %t0 = fmul fast float %a, %b %t1 = fmul fast float %c, %d %t2 = fadd fast float %t0, %t1 @@ -183,14 +261,25 @@ define float @fmac_sequence_simple(float %a, float %b, float %c, float %d, float } define float @fmac_sequence_innermost_fmul(float %a, float %b, float %c, float %d, float %e, float %f, float %g) #0 { -; GCN-LABEL: fmac_sequence_innermost_fmul: -; GCN: ; %bb.0: -; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: v_mad_f32 v2, v2, v3, v6 -; GCN-NEXT: v_fmac_f32_e32 v2, v0, v1 -; GCN-NEXT: v_fmac_f32_e32 v2, v4, v5 -; GCN-NEXT: v_mov_b32_e32 v0, v2 -; GCN-NEXT: s_setpc_b64 s[30:31] +; GFX10-LABEL: fmac_sequence_innermost_fmul: +; GFX10: ; %bb.0: +; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: v_mad_f32 v2, v2, v3, v6 +; GFX10-NEXT: v_fmac_f32_e32 v2, v0, v1 +; GFX10-NEXT: v_fmac_f32_e32 v2, v4, v5 +; GFX10-NEXT: v_mov_b32_e32 v0, v2 +; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: fmac_sequence_innermost_fmul: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_fma_f32 v2, v2, v3, v6 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_fmac_f32_e32 v2, v0, v1 +; GFX11-NEXT: v_fmac_f32_e32 v2, v4, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v0, v2 +; GFX11-NEXT: s_setpc_b64 s[30:31] %t0 = fmul fast float %a, %b %t1 = fmul fast float %c, %d %t2 = fadd fast float %t0, %t1 @@ -201,14 +290,25 @@ define float @fmac_sequence_innermost_fmul(float %a, float %b, float %c, float % } define float @fmac_sequence_innermost_fmul_swapped_operands(float %a, float %b, float %c, float %d, float %e, float %f, float %g) #0 { -; GCN-LABEL: fmac_sequence_innermost_fmul_swapped_operands: -; GCN: ; %bb.0: -; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: v_mad_f32 v2, v2, v3, v6 -; GCN-NEXT: v_fmac_f32_e32 v2, v0, v1 -; GCN-NEXT: v_fmac_f32_e32 v2, v4, v5 -; GCN-NEXT: v_mov_b32_e32 v0, v2 -; GCN-NEXT: s_setpc_b64 s[30:31] +; GFX10-LABEL: fmac_sequence_innermost_fmul_swapped_operands: +; GFX10: ; %bb.0: +; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: v_mad_f32 v2, v2, v3, v6 +; GFX10-NEXT: v_fmac_f32_e32 v2, v0, v1 +; GFX10-NEXT: v_fmac_f32_e32 v2, v4, v5 +; GFX10-NEXT: v_mov_b32_e32 v0, v2 +; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: fmac_sequence_innermost_fmul_swapped_operands: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_fma_f32 v2, v2, v3, v6 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_fmac_f32_e32 v2, v0, v1 +; GFX11-NEXT: v_fmac_f32_e32 v2, v4, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v0, v2 +; GFX11-NEXT: s_setpc_b64 s[30:31] %t0 = fmul fast float %a, %b %t1 = fmul fast float %c, %d %t2 = fadd fast float %t0, %t1 @@ -219,12 +319,20 @@ define float @fmac_sequence_innermost_fmul_swapped_operands(float %a, float %b, } define amdgpu_ps float @fmac_sequence_innermost_fmul_sgpr(float inreg %a, float inreg %b, float inreg %c, float inreg %d, float inreg %e, float inreg %f, float %g) #0 { -; GCN-LABEL: fmac_sequence_innermost_fmul_sgpr: -; GCN: ; %bb.0: -; GCN-NEXT: v_mac_f32_e64 v0, s2, s3 -; GCN-NEXT: v_fmac_f32_e64 v0, s0, s1 -; GCN-NEXT: v_fmac_f32_e64 v0, s4, s5 -; GCN-NEXT: ; return to shader part epilog +; GFX10-LABEL: fmac_sequence_innermost_fmul_sgpr: +; GFX10: ; %bb.0: +; GFX10-NEXT: v_mac_f32_e64 v0, s2, s3 +; GFX10-NEXT: v_fmac_f32_e64 v0, s0, s1 +; GFX10-NEXT: v_fmac_f32_e64 v0, s4, s5 +; GFX10-NEXT: ; return to shader part epilog +; +; GFX11-LABEL: fmac_sequence_innermost_fmul_sgpr: +; GFX11: ; %bb.0: +; GFX11-NEXT: v_fmac_f32_e64 v0, s2, s3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_fmac_f32_e64 v0, s0, s1 +; GFX11-NEXT: v_fmac_f32_e64 v0, s4, s5 +; GFX11-NEXT: ; return to shader part epilog %t0 = fmul fast float %a, %b %t1 = fmul fast float %c, %d %t2 = fadd fast float %t0, %t1 @@ -235,14 +343,25 @@ define amdgpu_ps float @fmac_sequence_innermost_fmul_sgpr(float inreg %a, float } define amdgpu_ps float @fmac_sequence_innermost_fmul_multiple_use(float inreg %a, float inreg %b, float inreg %c, float inreg %d, float inreg %e, float inreg %f, float %g) #0 { -; GCN-LABEL: fmac_sequence_innermost_fmul_multiple_use: -; GCN: ; %bb.0: -; GCN-NEXT: v_mul_f32_e64 v1, s2, s3 -; GCN-NEXT: v_fmac_f32_e64 v1, s0, s1 -; GCN-NEXT: v_fma_f32 v2, s5, s4, v1 -; GCN-NEXT: v_fmac_f32_e32 v1, s5, v2 -; GCN-NEXT: v_add_f32_e32 v0, v1, v0 -; GCN-NEXT: ; return to shader part epilog +; GFX10-LABEL: fmac_sequence_innermost_fmul_multiple_use: +; GFX10: ; %bb.0: +; GFX10-NEXT: v_mul_f32_e64 v1, s2, s3 +; GFX10-NEXT: v_fmac_f32_e64 v1, s0, s1 +; GFX10-NEXT: v_fma_f32 v2, s5, s4, v1 +; GFX10-NEXT: v_fmac_f32_e32 v1, s5, v2 +; GFX10-NEXT: v_add_f32_e32 v0, v1, v0 +; GFX10-NEXT: ; return to shader part epilog +; +; GFX11-LABEL: fmac_sequence_innermost_fmul_multiple_use: +; GFX11: ; %bb.0: +; GFX11-NEXT: v_mul_f32_e64 v1, s2, s3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_fmac_f32_e64 v1, s0, s1 +; GFX11-NEXT: v_fma_f32 v2, s5, s4, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_fmac_f32_e32 v1, s5, v2 +; GFX11-NEXT: v_add_f32_e32 v0, v1, v0 +; GFX11-NEXT: ; return to shader part epilog %t0 = fmul fast float %a, %b %t1 = fmul fast float %c, %d %t2 = fadd fast float %t0, %t1 @@ -258,12 +377,20 @@ define amdgpu_ps float @fmac_sequence_innermost_fmul_multiple_use(float inreg %a ; selecting it as a multiply. In some cases the multiply is better because ; SIFoldOperands can fold it into a previous instruction as an output modifier. define amdgpu_ps float @fma_vs_output_modifier(float %x, i32 %n) #0 { -; GCN-LABEL: fma_vs_output_modifier: -; GCN: ; %bb.0: -; GCN-NEXT: v_cvt_f32_i32_e64 v1, v1 mul:2 -; GCN-NEXT: v_mul_f32_e32 v0, v0, v0 -; GCN-NEXT: v_mul_f32_e32 v0, v0, v1 -; GCN-NEXT: ; return to shader part epilog +; GFX10-LABEL: fma_vs_output_modifier: +; GFX10: ; %bb.0: +; GFX10-NEXT: v_cvt_f32_i32_e64 v1, v1 mul:2 +; GFX10-NEXT: v_mul_f32_e32 v0, v0, v0 +; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX10-NEXT: ; return to shader part epilog +; +; GFX11-LABEL: fma_vs_output_modifier: +; GFX11: ; %bb.0: +; GFX11-NEXT: v_cvt_f32_i32_e64 v1, v1 mul:2 +; GFX11-NEXT: v_mul_f32_e32 v0, v0, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-NEXT: ; return to shader part epilog %s = sitofp i32 %n to float %m = fmul contract float %x, %x %a = fmul contract float %m, 2.0 diff --git a/llvm/test/CodeGen/AMDGPU/fma.f16.ll b/llvm/test/CodeGen/AMDGPU/fma.f16.ll index 23971f2b681cb0c..1b7e19193bc6d11 100644 --- a/llvm/test/CodeGen/AMDGPU/fma.f16.ll +++ b/llvm/test/CodeGen/AMDGPU/fma.f16.ll @@ -3,6 +3,8 @@ ; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx900 < %s | FileCheck %s -check-prefixes=GFX9,GFX9-GISEL ; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1010 < %s | FileCheck %s -check-prefixes=GFX10,GFX10-SDAG ; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1010 < %s | FileCheck %s -check-prefixes=GFX10,GFX10-GISEL +; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1100 < %s | FileCheck %s -check-prefixes=GFX11,GFX11-SDAG +; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1100 < %s | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL declare half @llvm.fma.f16(half, half, half) declare half @llvm.maxnum.f16(half, half) @@ -19,6 +21,12 @@ define half @test_fma(half %x, half %y, half %z) { ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: v_fma_f16 v0, v0, v1, v2 ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: test_fma: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_fma_f16 v0, v0, v1, v2 +; GFX11-NEXT: s_setpc_b64 s[30:31] %r = call half @llvm.fma.f16(half %x, half %y, half %z) ret half %r } @@ -36,6 +44,12 @@ define half @test_fmac(half %x, half %y, half %z) { ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: v_fmac_f16_e32 v0, v1, v2 ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: test_fmac: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_fmac_f16_e32 v0, v1, v2 +; GFX11-NEXT: s_setpc_b64 s[30:31] %r = call half @llvm.fma.f16(half %y, half %z, half %x) ret half %r } @@ -61,6 +75,12 @@ define half @test_fmaak(half %x, half %y, half %z) { ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: v_fmaak_f16 v0, v0, v1, 0x4200 ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: test_fmaak: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_fmaak_f16 v0, v0, v1, 0x4200 +; GFX11-NEXT: s_setpc_b64 s[30:31] %r = call half @llvm.fma.f16(half %x, half %y, half 0xH4200) ret half %r } @@ -86,6 +106,12 @@ define half @test_fmamk(half %x, half %y, half %z) { ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: v_fmamk_f16 v0, v0, 0x4200, v2 ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: test_fmamk: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_fmamk_f16 v0, v0, 0x4200, v2 +; GFX11-NEXT: s_setpc_b64 s[30:31] %r = call half @llvm.fma.f16(half %x, half 0xH4200, half %z) ret half %r } @@ -139,6 +165,33 @@ define i32 @test_D139469_f16(half %arg) { ; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4 ; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s4 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: test_D139469_f16: +; GFX11-SDAG: ; %bb.0: ; %bb +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 0x211e +; GFX11-SDAG-NEXT: v_mul_f16_e32 v2, 0x291e, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_fmac_f16_e32 v1, 0x291e, v0 +; GFX11-SDAG-NEXT: v_min_f16_e32 v0, v2, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v0 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: test_D139469_f16: +; GFX11-GISEL: ; %bb.0: ; %bb +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: s_movk_i32 s0, 0x291e +; GFX11-GISEL-NEXT: v_mul_f16_e32 v1, 0x291e, v0 +; GFX11-GISEL-NEXT: v_fmaak_f16 v0, s0, v0, 0x211e +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v1 +; GFX11-GISEL-NEXT: v_cmp_gt_f16_e64 s0, 0, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) +; GFX11-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] bb: %i = fmul contract half %arg, 0xH291E %i1 = fcmp olt half %i, 0xH0000 @@ -213,6 +266,44 @@ define <2 x i32> @test_D139469_v2f16(<2 x half> %arg) { ; GFX10-GISEL-NEXT: s_or_b32 s4, s6, s5 ; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s4 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: test_D139469_v2f16: +; GFX11-SDAG: ; %bb.0: ; %bb +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: s_movk_i32 s0, 0x211e +; GFX11-SDAG-NEXT: v_pk_mul_f16 v1, 0x291e, v0 op_sel_hi:[0,1] +; GFX11-SDAG-NEXT: v_pk_fma_f16 v0, 0x291e, v0, s0 op_sel_hi:[0,1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_min_f16 v0, v1, v0 +; GFX11-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0 +; GFX11-SDAG-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v0 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v1 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: test_D139469_v2f16: +; GFX11-GISEL: ; %bb.0: ; %bb +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: s_mov_b32 s0, 0x291e291e +; GFX11-GISEL-NEXT: v_pk_mul_f16 v1, v0, 0x291e op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_fma_f16 v0, v0, s0, 0x211e op_sel_hi:[1,1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v1 +; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0 +; GFX11-GISEL-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v1 +; GFX11-GISEL-NEXT: v_cmp_gt_f16_e64 s0, 0, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_cmp_gt_f16_e64 s1, 0, v2 +; GFX11-GISEL-NEXT: v_cmp_gt_f16_e64 s2, 0, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(SALU_CYCLE_1) +; GFX11-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(SALU_CYCLE_1) +; GFX11-GISEL-NEXT: s_or_b32 s0, s1, s2 +; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] bb: %i = fmul contract <2 x half> %arg, %i1 = fcmp olt <2 x half> %i,