From e56bc14d48ee9371edf6442b6c38b82e9a70c781 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 15 Nov 2024 12:36:48 -0800 Subject: [PATCH] Integrate LLVM at llvm/llvm-project@b3134fa23383 Updates LLVM usage to match [b3134fa23383](https://github.com/llvm/llvm-project/commit/b3134fa23383) PiperOrigin-RevId: 696969446 --- third_party/llvm/generated.patch | 703 +++++++++++++++++++++++++++++++ third_party/llvm/workspace.bzl | 4 +- 2 files changed, 705 insertions(+), 2 deletions(-) diff --git a/third_party/llvm/generated.patch b/third_party/llvm/generated.patch index 509398da9..0113efadd 100644 --- a/third_party/llvm/generated.patch +++ b/third_party/llvm/generated.patch @@ -1 +1,704 @@ Auto generated patch. Do not edit or delete it, even if empty. +diff -ruN --strip-trailing-cr a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td +--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td ++++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td +@@ -739,20 +739,6 @@ + def CVT_f16x2_e5m2x2 : CVT_f16x2_fp8<"e5m2">; + } + +-def fpround_oneuse : PatFrag<(ops node:$a), (fpround node:$a), [{ +- return N->hasOneUse(); +-}]>; +- +-def : Pat<(v2bf16 (build_vector (bf16 (fpround_oneuse Float32Regs:$a)), +- (bf16 (fpround_oneuse Float32Regs:$b)))), +- (CVT_bf16x2_f32 Float32Regs:$a, Float32Regs:$b, CvtRN)>, +- Requires<[hasPTX<70>, hasSM<80>, hasBF16Math]>; +- +-def : Pat<(v2f16 (build_vector (f16 (fpround_oneuse Float32Regs:$a)), +- (f16 (fpround_oneuse Float32Regs:$b)))), +- (CVT_f16x2_f32 Float32Regs:$a, Float32Regs:$b, CvtRN)>, +- Requires<[hasPTX<70>, hasSM<80>, useFP16Math]>; +- + //----------------------------------- + // Selection instructions (selp) + //----------------------------------- +diff -ruN --strip-trailing-cr a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll +--- a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll ++++ b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll +@@ -204,7 +204,7 @@ + ; + ; SM80-LABEL: test_faddx2( + ; SM80: { +-; SM80-NEXT: .reg .b16 %rs<5>; ++; SM80-NEXT: .reg .b16 %rs<7>; + ; SM80-NEXT: .reg .b32 %r<4>; + ; SM80-NEXT: .reg .f32 %f<7>; + ; SM80-EMPTY: +@@ -216,16 +216,18 @@ + ; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1; + ; SM80-NEXT: cvt.f32.bf16 %f2, %rs4; + ; SM80-NEXT: add.rn.f32 %f3, %f2, %f1; ++; SM80-NEXT: cvt.rn.bf16.f32 %rs5, %f3; + ; SM80-NEXT: cvt.f32.bf16 %f4, %rs1; + ; SM80-NEXT: cvt.f32.bf16 %f5, %rs3; + ; SM80-NEXT: add.rn.f32 %f6, %f5, %f4; +-; SM80-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3; ++; SM80-NEXT: cvt.rn.bf16.f32 %rs6, %f6; ++; SM80-NEXT: mov.b32 %r3, {%rs6, %rs5}; + ; SM80-NEXT: st.param.b32 [func_retval0], %r3; + ; SM80-NEXT: ret; + ; + ; SM80-FTZ-LABEL: test_faddx2( + ; SM80-FTZ: { +-; SM80-FTZ-NEXT: .reg .b16 %rs<5>; ++; SM80-FTZ-NEXT: .reg .b16 %rs<7>; + ; SM80-FTZ-NEXT: .reg .b32 %r<4>; + ; SM80-FTZ-NEXT: .reg .f32 %f<7>; + ; SM80-FTZ-EMPTY: +@@ -237,10 +239,12 @@ + ; SM80-FTZ-NEXT: mov.b32 {%rs3, %rs4}, %r1; + ; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f2, %rs4; + ; SM80-FTZ-NEXT: add.rn.ftz.f32 %f3, %f2, %f1; ++; SM80-FTZ-NEXT: cvt.rn.bf16.f32 %rs5, %f3; + ; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f4, %rs1; + ; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f5, %rs3; + ; SM80-FTZ-NEXT: add.rn.ftz.f32 %f6, %f5, %f4; +-; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3; ++; SM80-FTZ-NEXT: cvt.rn.bf16.f32 %rs6, %f6; ++; SM80-FTZ-NEXT: mov.b32 %r3, {%rs6, %rs5}; + ; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r3; + ; SM80-FTZ-NEXT: ret; + ; +@@ -307,7 +311,7 @@ + ; + ; SM80-LABEL: test_fsubx2( + ; SM80: { +-; SM80-NEXT: .reg .b16 %rs<5>; ++; SM80-NEXT: .reg .b16 %rs<7>; + ; SM80-NEXT: .reg .b32 %r<4>; + ; SM80-NEXT: .reg .f32 %f<7>; + ; SM80-EMPTY: +@@ -319,16 +323,18 @@ + ; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1; + ; SM80-NEXT: cvt.f32.bf16 %f2, %rs4; + ; SM80-NEXT: sub.rn.f32 %f3, %f2, %f1; ++; SM80-NEXT: cvt.rn.bf16.f32 %rs5, %f3; + ; SM80-NEXT: cvt.f32.bf16 %f4, %rs1; + ; SM80-NEXT: cvt.f32.bf16 %f5, %rs3; + ; SM80-NEXT: sub.rn.f32 %f6, %f5, %f4; +-; SM80-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3; ++; SM80-NEXT: cvt.rn.bf16.f32 %rs6, %f6; ++; SM80-NEXT: mov.b32 %r3, {%rs6, %rs5}; + ; SM80-NEXT: st.param.b32 [func_retval0], %r3; + ; SM80-NEXT: ret; + ; + ; SM80-FTZ-LABEL: test_fsubx2( + ; SM80-FTZ: { +-; SM80-FTZ-NEXT: .reg .b16 %rs<5>; ++; SM80-FTZ-NEXT: .reg .b16 %rs<7>; + ; SM80-FTZ-NEXT: .reg .b32 %r<4>; + ; SM80-FTZ-NEXT: .reg .f32 %f<7>; + ; SM80-FTZ-EMPTY: +@@ -340,10 +346,12 @@ + ; SM80-FTZ-NEXT: mov.b32 {%rs3, %rs4}, %r1; + ; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f2, %rs4; + ; SM80-FTZ-NEXT: sub.rn.ftz.f32 %f3, %f2, %f1; ++; SM80-FTZ-NEXT: cvt.rn.bf16.f32 %rs5, %f3; + ; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f4, %rs1; + ; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f5, %rs3; + ; SM80-FTZ-NEXT: sub.rn.ftz.f32 %f6, %f5, %f4; +-; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3; ++; SM80-FTZ-NEXT: cvt.rn.bf16.f32 %rs6, %f6; ++; SM80-FTZ-NEXT: mov.b32 %r3, {%rs6, %rs5}; + ; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r3; + ; SM80-FTZ-NEXT: ret; + ; +@@ -410,7 +418,7 @@ + ; + ; SM80-LABEL: test_fmulx2( + ; SM80: { +-; SM80-NEXT: .reg .b16 %rs<5>; ++; SM80-NEXT: .reg .b16 %rs<7>; + ; SM80-NEXT: .reg .b32 %r<4>; + ; SM80-NEXT: .reg .f32 %f<7>; + ; SM80-EMPTY: +@@ -422,16 +430,18 @@ + ; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1; + ; SM80-NEXT: cvt.f32.bf16 %f2, %rs4; + ; SM80-NEXT: mul.rn.f32 %f3, %f2, %f1; ++; SM80-NEXT: cvt.rn.bf16.f32 %rs5, %f3; + ; SM80-NEXT: cvt.f32.bf16 %f4, %rs1; + ; SM80-NEXT: cvt.f32.bf16 %f5, %rs3; + ; SM80-NEXT: mul.rn.f32 %f6, %f5, %f4; +-; SM80-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3; ++; SM80-NEXT: cvt.rn.bf16.f32 %rs6, %f6; ++; SM80-NEXT: mov.b32 %r3, {%rs6, %rs5}; + ; SM80-NEXT: st.param.b32 [func_retval0], %r3; + ; SM80-NEXT: ret; + ; + ; SM80-FTZ-LABEL: test_fmulx2( + ; SM80-FTZ: { +-; SM80-FTZ-NEXT: .reg .b16 %rs<5>; ++; SM80-FTZ-NEXT: .reg .b16 %rs<7>; + ; SM80-FTZ-NEXT: .reg .b32 %r<4>; + ; SM80-FTZ-NEXT: .reg .f32 %f<7>; + ; SM80-FTZ-EMPTY: +@@ -443,10 +453,12 @@ + ; SM80-FTZ-NEXT: mov.b32 {%rs3, %rs4}, %r1; + ; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f2, %rs4; + ; SM80-FTZ-NEXT: mul.rn.ftz.f32 %f3, %f2, %f1; ++; SM80-FTZ-NEXT: cvt.rn.bf16.f32 %rs5, %f3; + ; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f4, %rs1; + ; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f5, %rs3; + ; SM80-FTZ-NEXT: mul.rn.ftz.f32 %f6, %f5, %f4; +-; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3; ++; SM80-FTZ-NEXT: cvt.rn.bf16.f32 %rs6, %f6; ++; SM80-FTZ-NEXT: mov.b32 %r3, {%rs6, %rs5}; + ; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r3; + ; SM80-FTZ-NEXT: ret; + ; +@@ -513,7 +525,7 @@ + ; + ; SM80-LABEL: test_fdiv( + ; SM80: { +-; SM80-NEXT: .reg .b16 %rs<5>; ++; SM80-NEXT: .reg .b16 %rs<7>; + ; SM80-NEXT: .reg .b32 %r<4>; + ; SM80-NEXT: .reg .f32 %f<7>; + ; SM80-EMPTY: +@@ -525,16 +537,18 @@ + ; SM80-NEXT: mov.b32 {%rs3, %rs4}, %r1; + ; SM80-NEXT: cvt.f32.bf16 %f2, %rs4; + ; SM80-NEXT: div.rn.f32 %f3, %f2, %f1; ++; SM80-NEXT: cvt.rn.bf16.f32 %rs5, %f3; + ; SM80-NEXT: cvt.f32.bf16 %f4, %rs1; + ; SM80-NEXT: cvt.f32.bf16 %f5, %rs3; + ; SM80-NEXT: div.rn.f32 %f6, %f5, %f4; +-; SM80-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3; ++; SM80-NEXT: cvt.rn.bf16.f32 %rs6, %f6; ++; SM80-NEXT: mov.b32 %r3, {%rs6, %rs5}; + ; SM80-NEXT: st.param.b32 [func_retval0], %r3; + ; SM80-NEXT: ret; + ; + ; SM80-FTZ-LABEL: test_fdiv( + ; SM80-FTZ: { +-; SM80-FTZ-NEXT: .reg .b16 %rs<5>; ++; SM80-FTZ-NEXT: .reg .b16 %rs<7>; + ; SM80-FTZ-NEXT: .reg .b32 %r<4>; + ; SM80-FTZ-NEXT: .reg .f32 %f<7>; + ; SM80-FTZ-EMPTY: +@@ -546,16 +560,18 @@ + ; SM80-FTZ-NEXT: mov.b32 {%rs3, %rs4}, %r1; + ; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f2, %rs4; + ; SM80-FTZ-NEXT: div.rn.ftz.f32 %f3, %f2, %f1; ++; SM80-FTZ-NEXT: cvt.rn.bf16.f32 %rs5, %f3; + ; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f4, %rs1; + ; SM80-FTZ-NEXT: cvt.ftz.f32.bf16 %f5, %rs3; + ; SM80-FTZ-NEXT: div.rn.ftz.f32 %f6, %f5, %f4; +-; SM80-FTZ-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3; ++; SM80-FTZ-NEXT: cvt.rn.bf16.f32 %rs6, %f6; ++; SM80-FTZ-NEXT: mov.b32 %r3, {%rs6, %rs5}; + ; SM80-FTZ-NEXT: st.param.b32 [func_retval0], %r3; + ; SM80-FTZ-NEXT: ret; + ; + ; SM90-LABEL: test_fdiv( + ; SM90: { +-; SM90-NEXT: .reg .b16 %rs<5>; ++; SM90-NEXT: .reg .b16 %rs<7>; + ; SM90-NEXT: .reg .b32 %r<4>; + ; SM90-NEXT: .reg .f32 %f<7>; + ; SM90-EMPTY: +@@ -567,10 +583,12 @@ + ; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r1; + ; SM90-NEXT: cvt.f32.bf16 %f2, %rs4; + ; SM90-NEXT: div.rn.f32 %f3, %f2, %f1; ++; SM90-NEXT: cvt.rn.bf16.f32 %rs5, %f3; + ; SM90-NEXT: cvt.f32.bf16 %f4, %rs1; + ; SM90-NEXT: cvt.f32.bf16 %f5, %rs3; + ; SM90-NEXT: div.rn.f32 %f6, %f5, %f4; +-; SM90-NEXT: cvt.rn.bf16x2.f32 %r3, %f6, %f3; ++; SM90-NEXT: cvt.rn.bf16.f32 %rs6, %f6; ++; SM90-NEXT: mov.b32 %r3, {%rs6, %rs5}; + ; SM90-NEXT: st.param.b32 [func_retval0], %r3; + ; SM90-NEXT: ret; + %r = fdiv <2 x bfloat> %a, %b +diff -ruN --strip-trailing-cr a/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll b/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll +--- a/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll ++++ b/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll +@@ -13,7 +13,9 @@ + ; CHECK-DAG: cvt.f32.bf16 [[AF1:%f[0-9]+]], [[A1]]; + ; CHECK-DAG: sin.approx.f32 [[RF0:%f[0-9]+]], [[AF0]]; + ; CHECK-DAG: sin.approx.f32 [[RF1:%f[0-9]+]], [[AF1]]; +-; CHECK: cvt.rn.bf16x2.f32 [[R:%r[0-9]+]], [[RF0]], [[RF1]] ++; CHECK-DAG: cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[RF0]]; ++; CHECK-DAG: cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[RF1]]; ++; CHECK: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]} + ; CHECK: st.param.b32 [func_retval0], [[R]]; + ; CHECK: ret; + define <2 x bfloat> @test_sin(<2 x bfloat> %a) #0 #1 { +@@ -28,7 +30,9 @@ + ; CHECK-DAG: cvt.f32.bf16 [[AF1:%f[0-9]+]], [[A1]]; + ; CHECK-DAG: cos.approx.f32 [[RF0:%f[0-9]+]], [[AF0]]; + ; CHECK-DAG: cos.approx.f32 [[RF1:%f[0-9]+]], [[AF1]]; +-; CHECK: cvt.rn.bf16x2.f32 [[R:%r[0-9]+]], [[RF0]], [[RF1]] ++; CHECK-DAG: cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[RF0]]; ++; CHECK-DAG: cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[RF1]]; ++; CHECK: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]} + ; CHECK: st.param.b32 [func_retval0], [[R]]; + ; CHECK: ret; + define <2 x bfloat> @test_cos(<2 x bfloat> %a) #0 #1 { +diff -ruN --strip-trailing-cr a/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll +--- a/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll ++++ b/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll +@@ -26,7 +26,9 @@ + ; SM80-DAG: cvt.f32.bf16 [[FA1:%f[0-9]+]], [[A1]] + ; SM80-DAG: add.rn.f32 [[FR0:%f[0-9]+]], [[FA0]], 0f3F800000; + ; SM80-DAG: add.rn.f32 [[FR1:%f[0-9]+]], [[FA1]], 0f40000000; +-; SM80-DAG: cvt.rn.bf16x2.f32 [[R:%r[0-9]+]], [[FR0]], [[FR1]]; ++; SM80-DAG: cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[FR0]] ++; SM80-DAG: cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[FR1]] ++; SM80-DAG: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]} + ; + ; CHECK-NEXT: st.param.b32 [func_retval0], [[R]]; + ; CHECK-NEXT: ret; +@@ -66,7 +68,9 @@ + ; SM80-DAG: cvt.f32.bf16 [[FB1:%f[0-9]+]], [[B1]]; + ; SM80-DAG: sub.rn.f32 [[FR0:%f[0-9]+]], [[FA0]], [[FB0]]; + ; SM80-DAG: sub.rn.f32 [[FR1:%f[0-9]+]], [[FA1]], [[FB1]]; +-; SM80-DAG: cvt.rn.bf16x2.f32 [[R:%r[0-9]+]], [[FR0]], [[FR1]]; ++; SM80-DAG: cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[FR0]]; ++; SM80-DAG: cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[FR1]]; ++; SM80: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]}; + + ; CHECK: st.param.b32 [func_retval0], [[R]]; + ; CHECK: ret; +@@ -89,7 +93,9 @@ + ; SM80-DAG: cvt.f32.bf16 [[FB1:%f[0-9]+]], [[B1]]; + ; SM80-DAG: mul.rn.f32 [[FR0:%f[0-9]+]], [[FA0]], [[FB0]]; + ; SM80-DAG: mul.rn.f32 [[FR1:%f[0-9]+]], [[FA1]], [[FB1]]; +-; SM80-DAG: cvt.rn.bf16x2.f32 [[R:%r[0-9]+]], [[FR0]], [[FR1]]; ++; SM80-DAG: cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[FR0]]; ++; SM80-DAG: cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[FR1]]; ++; SM80: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]}; + + ; CHECK: st.param.b32 [func_retval0], [[R]]; + ; CHECK: ret; +@@ -110,7 +116,9 @@ + ; CHECK-DAG: cvt.f32.bf16 [[FB1:%f[0-9]+]], [[B1]]; + ; CHECK-DAG: div.rn.f32 [[FR0:%f[0-9]+]], [[FA0]], [[FB0]]; + ; CHECK-DAG: div.rn.f32 [[FR1:%f[0-9]+]], [[FA1]], [[FB1]]; +-; CHECK: cvt.rn.bf16x2.f32 [[R:%r[0-9]+]], [[FR0]], [[FR1]]; ++; CHECK-DAG: cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[FR0]]; ++; CHECK-DAG: cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[FR1]]; ++; CHECK-NEXT: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]} + ; CHECK-NEXT: st.param.b32 [func_retval0], [[R]]; + ; CHECK-NEXT: ret; + +@@ -279,7 +287,9 @@ + + ; CHECK-LABEL: test_fptrunc_2xfloat( + ; CHECK: ld.param.v2.f32 {[[A0:%f[0-9]+]], [[A1:%f[0-9]+]]}, [test_fptrunc_2xfloat_param_0]; +-; CHECK: cvt.rn.bf16x2.f32 [[R:%r[0-9]+]], [[A0]], [[A1]]; ++; CHECK-DAG: cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[A0]]; ++; CHECK-DAG: cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[A1]]; ++; CHECK: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]} + ; CHECK: st.param.b32 [func_retval0], [[R]]; + ; CHECK: ret; + define <2 x bfloat> @test_fptrunc_2xfloat(<2 x float> %a) #0 { +@@ -349,7 +359,9 @@ + ; CHECK-DAG: cvt.f32.bf16 [[AF1:%f[0-9]+]], [[A1]]; + ; CHECK-DAG: sqrt.rn.f32 [[RF0:%f[0-9]+]], [[AF0]]; + ; CHECK-DAG: sqrt.rn.f32 [[RF1:%f[0-9]+]], [[AF1]]; +-; CHECK-DAG: cvt.rn.bf16x2.f32 [[R:%r[0-9]+]], [[RF0]], [[RF1]]; ++; CHECK-DAG: cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[RF0]]; ++; CHECK-DAG: cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[RF1]]; ++; CHECK: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]} + ; CHECK: st.param.b32 [func_retval0], [[R]]; + ; CHECK: ret; + define <2 x bfloat> @test_sqrt(<2 x bfloat> %a) #0 { +@@ -424,7 +436,9 @@ + ; SM80-DAG: cvt.f32.bf16 [[FA1:%f[0-9]+]], [[A1]]; + ; SM80-DAG: cvt.rmi.f32.f32 [[RF0:%f[0-9]+]], [[FA0]]; + ; SM80-DAG: cvt.rmi.f32.f32 [[RF1:%f[0-9]+]], [[FA1]]; +-; SM80: cvt.rn.bf16x2.f32 [[R:%r[0-9]+]], [[RF0]], [[RF1]]; ++; SM80-DAG: cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[RF0]]; ++; SM80-DAG: cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[RF1]]; ++; CHECK: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]} + ; CHECK: st.param.b32 [func_retval0], [[R]]; + ; CHECK: ret; + define <2 x bfloat> @test_floor(<2 x bfloat> %a) #0 { +@@ -441,7 +455,9 @@ + ; SM80-DAG: cvt.f32.bf16 [[FA1:%f[0-9]+]], [[A1]]; + ; SM80-DAG: cvt.rpi.f32.f32 [[RF0:%f[0-9]+]], [[FA0]]; + ; SM80-DAG: cvt.rpi.f32.f32 [[RF1:%f[0-9]+]], [[FA1]]; +-; SM80: cvt.rn.bf16x2.f32 [[R:%r[0-9]+]], [[RF0]], [[RF1]]; ++; SM80-DAG: cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[RF0]]; ++; SM80-DAG: cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[RF1]]; ++; CHECK: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]} + ; CHECK: st.param.b32 [func_retval0], [[R]]; + ; CHECK: ret; + define <2 x bfloat> @test_ceil(<2 x bfloat> %a) #0 { +@@ -454,7 +470,7 @@ + ; CHECK-DAG: mov.b32 {[[A0:%rs[0-9]+]], [[A1:%rs[0-9]+]]}, [[A]]; + ; SM90: cvt.rzi.bf16.bf16 [[R1:%rs[0-9]+]], [[A1]]; + ; SM90: cvt.rzi.bf16.bf16 [[R0:%rs[0-9]+]], [[A0]]; +-; SM90: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]} ++; CHECK: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]} + ; CHECK: st.param.b32 [func_retval0], [[R]]; + ; CHECK: ret; + define <2 x bfloat> @test_trunc(<2 x bfloat> %a) #0 { +@@ -467,7 +483,7 @@ + ; CHECK-DAG: mov.b32 {[[A0:%rs[0-9]+]], [[A1:%rs[0-9]+]]}, [[A]]; + ; SM90: cvt.rni.bf16.bf16 [[R1:%rs[0-9]+]], [[A1]]; + ; SM90: cvt.rni.bf16.bf16 [[R0:%rs[0-9]+]], [[A0]]; +-; SM90: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]} ++; CHECK: mov.b32 [[R:%r[0-9]+]], {[[R0]], [[R1]]} + ; CHECK: st.param.b32 [func_retval0], [[R]]; + ; CHECK: ret; + define <2 x bfloat> @test_rint(<2 x bfloat> %a) #0 { +diff -ruN --strip-trailing-cr a/llvm/test/CodeGen/NVPTX/convert-sm80.ll b/llvm/test/CodeGen/NVPTX/convert-sm80.ll +--- a/llvm/test/CodeGen/NVPTX/convert-sm80.ll ++++ b/llvm/test/CodeGen/NVPTX/convert-sm80.ll +@@ -1,70 +1,41 @@ +-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 + ; RUN: llc < %s -march=nvptx64 -mcpu=sm_80 -mattr=+ptx70 | FileCheck %s + ; RUN: %if ptxas-11.0 %{ llc < %s -march=nvptx64 -mcpu=sm_80 -mattr=+ptx70 | %ptxas-verify -arch=sm_80 %} + + ++; CHECK-LABEL: cvt_rn_bf16x2_f32 + define <2 x bfloat> @cvt_rn_bf16x2_f32(float %f1, float %f2) { +-; CHECK-LABEL: cvt_rn_bf16x2_f32( +-; CHECK: { +-; CHECK-NEXT: .reg .b32 %r<2>; +-; CHECK-NEXT: .reg .f32 %f<3>; +-; CHECK-EMPTY: +-; CHECK-NEXT: // %bb.0: +-; CHECK-NEXT: ld.param.f32 %f1, [cvt_rn_bf16x2_f32_param_0]; +-; CHECK-NEXT: ld.param.f32 %f2, [cvt_rn_bf16x2_f32_param_1]; +-; CHECK-NEXT: cvt.rn.bf16x2.f32 %r1, %f1, %f2; +-; CHECK-NEXT: st.param.b32 [func_retval0], %r1; +-; CHECK-NEXT: ret; +- %val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rn(float %f1, float %f2) +- ret <2 x bfloat> %val ++ ++; CHECK: cvt.rn.bf16x2.f32 ++ %val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rn(float %f1, float %f2); ++ ++ret <2 x bfloat> %val + } + ++; CHECK-LABEL: cvt_rn_relu_bf16x2_f32 + define <2 x bfloat> @cvt_rn_relu_bf16x2_f32(float %f1, float %f2) { +-; CHECK-LABEL: cvt_rn_relu_bf16x2_f32( +-; CHECK: { +-; CHECK-NEXT: .reg .b32 %r<2>; +-; CHECK-NEXT: .reg .f32 %f<3>; +-; CHECK-EMPTY: +-; CHECK-NEXT: // %bb.0: +-; CHECK-NEXT: ld.param.f32 %f1, [cvt_rn_relu_bf16x2_f32_param_0]; +-; CHECK-NEXT: ld.param.f32 %f2, [cvt_rn_relu_bf16x2_f32_param_1]; +-; CHECK-NEXT: cvt.rn.relu.bf16x2.f32 %r1, %f1, %f2; +-; CHECK-NEXT: st.param.b32 [func_retval0], %r1; +-; CHECK-NEXT: ret; +- %val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rn.relu(float %f1, float %f2) +- ret <2 x bfloat> %val ++ ++; CHECK: cvt.rn.relu.bf16x2.f32 ++%val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rn.relu(float %f1, float %f2); ++ ++ret <2 x bfloat> %val + } + ++; CHECK-LABEL: cvt_rz_bf16x2_f32 + define <2 x bfloat> @cvt_rz_bf16x2_f32(float %f1, float %f2) { +-; CHECK-LABEL: cvt_rz_bf16x2_f32( +-; CHECK: { +-; CHECK-NEXT: .reg .b32 %r<2>; +-; CHECK-NEXT: .reg .f32 %f<3>; +-; CHECK-EMPTY: +-; CHECK-NEXT: // %bb.0: +-; CHECK-NEXT: ld.param.f32 %f1, [cvt_rz_bf16x2_f32_param_0]; +-; CHECK-NEXT: ld.param.f32 %f2, [cvt_rz_bf16x2_f32_param_1]; +-; CHECK-NEXT: cvt.rz.bf16x2.f32 %r1, %f1, %f2; +-; CHECK-NEXT: st.param.b32 [func_retval0], %r1; +-; CHECK-NEXT: ret; +- %val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %f1, float %f2) +- ret <2 x bfloat> %val ++ ++; CHECK: cvt.rz.bf16x2.f32 ++ %val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %f1, float %f2); ++ ++ret <2 x bfloat> %val + } + ++; CHECK-LABEL: cvt_rz_relu_bf16x2_f32 + define <2 x bfloat> @cvt_rz_relu_bf16x2_f32(float %f1, float %f2) { +-; CHECK-LABEL: cvt_rz_relu_bf16x2_f32( +-; CHECK: { +-; CHECK-NEXT: .reg .b32 %r<2>; +-; CHECK-NEXT: .reg .f32 %f<3>; +-; CHECK-EMPTY: +-; CHECK-NEXT: // %bb.0: +-; CHECK-NEXT: ld.param.f32 %f1, [cvt_rz_relu_bf16x2_f32_param_0]; +-; CHECK-NEXT: ld.param.f32 %f2, [cvt_rz_relu_bf16x2_f32_param_1]; +-; CHECK-NEXT: cvt.rz.relu.bf16x2.f32 %r1, %f1, %f2; +-; CHECK-NEXT: st.param.b32 [func_retval0], %r1; +-; CHECK-NEXT: ret; +- %val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz.relu(float %f1, float %f2) +- ret <2 x bfloat> %val ++ ++; CHECK: cvt.rz.relu.bf16x2.f32 ++%val = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz.relu(float %f1, float %f2); ++ ++ret <2 x bfloat> %val + } + + declare <2 x bfloat> @llvm.nvvm.ff2bf16x2.rn(float, float) +@@ -72,68 +43,40 @@ + declare <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float, float) + declare <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz.relu(float, float) + ++; CHECK-LABEL: cvt_rn_f16x2_f32 + define <2 x half> @cvt_rn_f16x2_f32(float %f1, float %f2) { +-; CHECK-LABEL: cvt_rn_f16x2_f32( +-; CHECK: { +-; CHECK-NEXT: .reg .b32 %r<2>; +-; CHECK-NEXT: .reg .f32 %f<3>; +-; CHECK-EMPTY: +-; CHECK-NEXT: // %bb.0: +-; CHECK-NEXT: ld.param.f32 %f1, [cvt_rn_f16x2_f32_param_0]; +-; CHECK-NEXT: ld.param.f32 %f2, [cvt_rn_f16x2_f32_param_1]; +-; CHECK-NEXT: cvt.rn.f16x2.f32 %r1, %f1, %f2; +-; CHECK-NEXT: st.param.b32 [func_retval0], %r1; +-; CHECK-NEXT: ret; +- %val = call <2 x half> @llvm.nvvm.ff2f16x2.rn(float %f1, float %f2) +- ret <2 x half> %val ++ ++; CHECK: cvt.rn.f16x2.f32 ++ %val = call <2 x half> @llvm.nvvm.ff2f16x2.rn(float %f1, float %f2); ++ ++ret <2 x half> %val + } + ++; CHECK-LABEL: cvt_rn_relu_f16x2_f32 + define <2 x half> @cvt_rn_relu_f16x2_f32(float %f1, float %f2) { +-; CHECK-LABEL: cvt_rn_relu_f16x2_f32( +-; CHECK: { +-; CHECK-NEXT: .reg .b32 %r<2>; +-; CHECK-NEXT: .reg .f32 %f<3>; +-; CHECK-EMPTY: +-; CHECK-NEXT: // %bb.0: +-; CHECK-NEXT: ld.param.f32 %f1, [cvt_rn_relu_f16x2_f32_param_0]; +-; CHECK-NEXT: ld.param.f32 %f2, [cvt_rn_relu_f16x2_f32_param_1]; +-; CHECK-NEXT: cvt.rn.relu.f16x2.f32 %r1, %f1, %f2; +-; CHECK-NEXT: st.param.b32 [func_retval0], %r1; +-; CHECK-NEXT: ret; +- %val = call <2 x half> @llvm.nvvm.ff2f16x2.rn.relu(float %f1, float %f2) +- ret <2 x half> %val ++ ++; CHECK: cvt.rn.relu.f16x2.f32 ++%val = call <2 x half> @llvm.nvvm.ff2f16x2.rn.relu(float %f1, float %f2); ++ ++ret <2 x half> %val + } + ++; CHECK-LABEL: cvt_rz_f16x2_f32 + define <2 x half> @cvt_rz_f16x2_f32(float %f1, float %f2) { +-; CHECK-LABEL: cvt_rz_f16x2_f32( +-; CHECK: { +-; CHECK-NEXT: .reg .b32 %r<2>; +-; CHECK-NEXT: .reg .f32 %f<3>; +-; CHECK-EMPTY: +-; CHECK-NEXT: // %bb.0: +-; CHECK-NEXT: ld.param.f32 %f1, [cvt_rz_f16x2_f32_param_0]; +-; CHECK-NEXT: ld.param.f32 %f2, [cvt_rz_f16x2_f32_param_1]; +-; CHECK-NEXT: cvt.rz.f16x2.f32 %r1, %f1, %f2; +-; CHECK-NEXT: st.param.b32 [func_retval0], %r1; +-; CHECK-NEXT: ret; +- %val = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %f1, float %f2) +- ret <2 x half> %val ++ ++; CHECK: cvt.rz.f16x2.f32 ++ %val = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %f1, float %f2); ++ ++ret <2 x half> %val + } + ++; CHECK-LABEL: cvt_rz_relu_f16x2_f32 + define <2 x half> @cvt_rz_relu_f16x2_f32(float %f1, float %f2) { +-; CHECK-LABEL: cvt_rz_relu_f16x2_f32( +-; CHECK: { +-; CHECK-NEXT: .reg .b32 %r<2>; +-; CHECK-NEXT: .reg .f32 %f<3>; +-; CHECK-EMPTY: +-; CHECK-NEXT: // %bb.0: +-; CHECK-NEXT: ld.param.f32 %f1, [cvt_rz_relu_f16x2_f32_param_0]; +-; CHECK-NEXT: ld.param.f32 %f2, [cvt_rz_relu_f16x2_f32_param_1]; +-; CHECK-NEXT: cvt.rz.relu.f16x2.f32 %r1, %f1, %f2; +-; CHECK-NEXT: st.param.b32 [func_retval0], %r1; +-; CHECK-NEXT: ret; +- %val = call <2 x half> @llvm.nvvm.ff2f16x2.rz.relu(float %f1, float %f2) +- ret <2 x half> %val ++ ++; CHECK: cvt.rz.relu.f16x2.f32 ++%val = call <2 x half> @llvm.nvvm.ff2f16x2.rz.relu(float %f1, float %f2); ++ ++ret <2 x half> %val + } + + declare <2 x half> @llvm.nvvm.ff2f16x2.rn(float, float) +@@ -141,64 +84,40 @@ + declare <2 x half> @llvm.nvvm.ff2f16x2.rz(float, float) + declare <2 x half> @llvm.nvvm.ff2f16x2.rz.relu(float, float) + ++; CHECK-LABEL: cvt_rn_bf16_f32 + define bfloat @cvt_rn_bf16_f32(float %f1) { +-; CHECK-LABEL: cvt_rn_bf16_f32( +-; CHECK: { +-; CHECK-NEXT: .reg .b16 %rs<2>; +-; CHECK-NEXT: .reg .f32 %f<2>; +-; CHECK-EMPTY: +-; CHECK-NEXT: // %bb.0: +-; CHECK-NEXT: ld.param.f32 %f1, [cvt_rn_bf16_f32_param_0]; +-; CHECK-NEXT: cvt.rn.bf16.f32 %rs1, %f1; +-; CHECK-NEXT: st.param.b16 [func_retval0], %rs1; +-; CHECK-NEXT: ret; +- %val = call bfloat @llvm.nvvm.f2bf16.rn(float %f1) +- ret bfloat %val ++ ++; CHECK: cvt.rn.bf16.f32 ++ %val = call bfloat @llvm.nvvm.f2bf16.rn(float %f1); ++ ++ret bfloat %val + } + ++; CHECK-LABEL: cvt_rn_relu_bf16_f32 + define bfloat @cvt_rn_relu_bf16_f32(float %f1) { +-; CHECK-LABEL: cvt_rn_relu_bf16_f32( +-; CHECK: { +-; CHECK-NEXT: .reg .b16 %rs<2>; +-; CHECK-NEXT: .reg .f32 %f<2>; +-; CHECK-EMPTY: +-; CHECK-NEXT: // %bb.0: +-; CHECK-NEXT: ld.param.f32 %f1, [cvt_rn_relu_bf16_f32_param_0]; +-; CHECK-NEXT: cvt.rn.relu.bf16.f32 %rs1, %f1; +-; CHECK-NEXT: st.param.b16 [func_retval0], %rs1; +-; CHECK-NEXT: ret; +- %val = call bfloat @llvm.nvvm.f2bf16.rn.relu(float %f1) +- ret bfloat %val ++ ++; CHECK: cvt.rn.relu.bf16.f32 ++%val = call bfloat @llvm.nvvm.f2bf16.rn.relu(float %f1); ++ ++ret bfloat %val + } + ++; CHECK-LABEL: cvt_rz_bf16_f32 + define bfloat @cvt_rz_bf16_f32(float %f1) { +-; CHECK-LABEL: cvt_rz_bf16_f32( +-; CHECK: { +-; CHECK-NEXT: .reg .b16 %rs<2>; +-; CHECK-NEXT: .reg .f32 %f<2>; +-; CHECK-EMPTY: +-; CHECK-NEXT: // %bb.0: +-; CHECK-NEXT: ld.param.f32 %f1, [cvt_rz_bf16_f32_param_0]; +-; CHECK-NEXT: cvt.rz.bf16.f32 %rs1, %f1; +-; CHECK-NEXT: st.param.b16 [func_retval0], %rs1; +-; CHECK-NEXT: ret; +- %val = call bfloat @llvm.nvvm.f2bf16.rz(float %f1) +- ret bfloat %val ++ ++; CHECK: cvt.rz.bf16.f32 ++ %val = call bfloat @llvm.nvvm.f2bf16.rz(float %f1); ++ ++ret bfloat %val + } + ++; CHECK-LABEL: cvt_rz_relu_bf16_f32 + define bfloat @cvt_rz_relu_bf16_f32(float %f1) { +-; CHECK-LABEL: cvt_rz_relu_bf16_f32( +-; CHECK: { +-; CHECK-NEXT: .reg .b16 %rs<2>; +-; CHECK-NEXT: .reg .f32 %f<2>; +-; CHECK-EMPTY: +-; CHECK-NEXT: // %bb.0: +-; CHECK-NEXT: ld.param.f32 %f1, [cvt_rz_relu_bf16_f32_param_0]; +-; CHECK-NEXT: cvt.rz.relu.bf16.f32 %rs1, %f1; +-; CHECK-NEXT: st.param.b16 [func_retval0], %rs1; +-; CHECK-NEXT: ret; +- %val = call bfloat @llvm.nvvm.f2bf16.rz.relu(float %f1) +- ret bfloat %val ++ ++; CHECK: cvt.rz.relu.bf16.f32 ++%val = call bfloat @llvm.nvvm.f2bf16.rz.relu(float %f1); ++ ++ret bfloat %val + } + + declare bfloat @llvm.nvvm.f2bf16.rn(float) +@@ -206,58 +125,13 @@ + declare bfloat @llvm.nvvm.f2bf16.rz(float) + declare bfloat @llvm.nvvm.f2bf16.rz.relu(float) + ++; CHECK-LABEL: cvt_rna_tf32_f32 + define i32 @cvt_rna_tf32_f32(float %f1) { +-; CHECK-LABEL: cvt_rna_tf32_f32( +-; CHECK: { +-; CHECK-NEXT: .reg .b32 %r<2>; +-; CHECK-NEXT: .reg .f32 %f<2>; +-; CHECK-EMPTY: +-; CHECK-NEXT: // %bb.0: +-; CHECK-NEXT: ld.param.f32 %f1, [cvt_rna_tf32_f32_param_0]; +-; CHECK-NEXT: cvt.rna.tf32.f32 %r1, %f1; +-; CHECK-NEXT: st.param.b32 [func_retval0], %r1; +-; CHECK-NEXT: ret; +- %val = call i32 @llvm.nvvm.f2tf32.rna(float %f1) +- ret i32 %val +-} + +-declare i32 @llvm.nvvm.f2tf32.rna(float) ++; CHECK: cvt.rna.tf32.f32 ++ %val = call i32 @llvm.nvvm.f2tf32.rna(float %f1); + +- +-define <2 x bfloat> @fold_ff2bf16x2(float %a, float %b) { +-; CHECK-LABEL: fold_ff2bf16x2( +-; CHECK: { +-; CHECK-NEXT: .reg .b32 %r<2>; +-; CHECK-NEXT: .reg .f32 %f<3>; +-; CHECK-EMPTY: +-; CHECK-NEXT: // %bb.0: +-; CHECK-NEXT: ld.param.f32 %f1, [fold_ff2bf16x2_param_0]; +-; CHECK-NEXT: ld.param.f32 %f2, [fold_ff2bf16x2_param_1]; +-; CHECK-NEXT: cvt.rn.bf16x2.f32 %r1, %f1, %f2; +-; CHECK-NEXT: st.param.b32 [func_retval0], %r1; +-; CHECK-NEXT: ret; +- %ah = fptrunc float %a to bfloat +- %bh = fptrunc float %b to bfloat +- %v0 = insertelement <2 x bfloat> poison, bfloat %ah, i64 0 +- %v1 = insertelement <2 x bfloat> %v0, bfloat %bh, i64 1 +- ret <2 x bfloat> %v1 +-} +- +-define <2 x half> @fold_ff2f16x2(float %a, float %b) { +-; CHECK-LABEL: fold_ff2f16x2( +-; CHECK: { +-; CHECK-NEXT: .reg .b32 %r<2>; +-; CHECK-NEXT: .reg .f32 %f<3>; +-; CHECK-EMPTY: +-; CHECK-NEXT: // %bb.0: +-; CHECK-NEXT: ld.param.f32 %f1, [fold_ff2f16x2_param_0]; +-; CHECK-NEXT: ld.param.f32 %f2, [fold_ff2f16x2_param_1]; +-; CHECK-NEXT: cvt.rn.f16x2.f32 %r1, %f1, %f2; +-; CHECK-NEXT: st.param.b32 [func_retval0], %r1; +-; CHECK-NEXT: ret; +- %ah = fptrunc float %a to half +- %bh = fptrunc float %b to half +- %v0 = insertelement <2 x half> poison, half %ah, i64 0 +- %v1 = insertelement <2 x half> %v0, half %bh, i64 1 +- ret <2 x half> %v1 ++ret i32 %val + } ++ ++declare i32 @llvm.nvvm.f2tf32.rna(float) diff --git a/third_party/llvm/workspace.bzl b/third_party/llvm/workspace.bzl index ad42b42a9..63355e174 100644 --- a/third_party/llvm/workspace.bzl +++ b/third_party/llvm/workspace.bzl @@ -4,8 +4,8 @@ load("//third_party:repo.bzl", "tf_http_archive") def repo(name): """Imports LLVM.""" - LLVM_COMMIT = "03730cdd3d10c5270fe436777a37d50b0838a3bf" - LLVM_SHA256 = "54d843249c75b200f7bf9b7947079fe16fa0b657c4aee4abdde4ac05a9cd5f84" + LLVM_COMMIT = "b3134fa2338388adf8cfb2d77339d0b042eab9f6" + LLVM_SHA256 = "b6024606092290b0838735c26ad1c5c239b3e931136b420af8680e3a1156e759" tf_http_archive( name = name,