-
Notifications
You must be signed in to change notification settings - Fork 26
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Integrate LLVM at llvm/llvm-project@9c80eb7c83c6
Updates LLVM usage to match [9c80eb7c83c6](llvm/llvm-project@9c80eb7c83c6) PiperOrigin-RevId: 686934080
- Loading branch information
1 parent
50cb4dd
commit 99035d5
Showing
2 changed files
with
2 additions
and
378 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,377 +1 @@ | ||
| Auto generated patch. Do not edit or delete it, even if empty. | ||
| diff -ruN --strip-trailing-cr a/libc/src/math/generic/issignaling.cpp b/libc/src/math/generic/issignaling.cpp | ||
| --- a/libc/src/math/generic/issignaling.cpp | ||
| +++ b/libc/src/math/generic/issignaling.cpp | ||
| @@ -14,7 +14,7 @@ | ||
| namespace LIBC_NAMESPACE_DECL { | ||
|
|
||
| LLVM_LIBC_FUNCTION(int, issignaling, (double x)) { | ||
| - return fputil::issignaling(x); | ||
| + return fputil::issignaling_impl(x); | ||
| } | ||
|
|
||
| } // namespace LIBC_NAMESPACE_DECL | ||
| diff -ruN --strip-trailing-cr a/libc/src/math/generic/issignalingf128.cpp b/libc/src/math/generic/issignalingf128.cpp | ||
| --- a/libc/src/math/generic/issignalingf128.cpp | ||
| +++ b/libc/src/math/generic/issignalingf128.cpp | ||
| @@ -14,7 +14,7 @@ | ||
| namespace LIBC_NAMESPACE_DECL { | ||
|
|
||
| LLVM_LIBC_FUNCTION(int, issignalingf128, (float128 x)) { | ||
| - return fputil::issignaling(x); | ||
| + return fputil::issignaling_impl(x); | ||
| } | ||
|
|
||
| } // namespace LIBC_NAMESPACE_DECL | ||
| diff -ruN --strip-trailing-cr a/libc/src/math/generic/issignalingf16.cpp b/libc/src/math/generic/issignalingf16.cpp | ||
| --- a/libc/src/math/generic/issignalingf16.cpp | ||
| +++ b/libc/src/math/generic/issignalingf16.cpp | ||
| @@ -14,7 +14,7 @@ | ||
| namespace LIBC_NAMESPACE_DECL { | ||
|
|
||
| LLVM_LIBC_FUNCTION(int, issignalingf16, (float16 x)) { | ||
| - return fputil::issignaling(x); | ||
| + return fputil::issignaling_impl(x); | ||
| } | ||
|
|
||
| } // namespace LIBC_NAMESPACE_DECL | ||
| diff -ruN --strip-trailing-cr a/libc/src/math/generic/issignalingf.cpp b/libc/src/math/generic/issignalingf.cpp | ||
| --- a/libc/src/math/generic/issignalingf.cpp | ||
| +++ b/libc/src/math/generic/issignalingf.cpp | ||
| @@ -14,7 +14,7 @@ | ||
| namespace LIBC_NAMESPACE_DECL { | ||
|
|
||
| LLVM_LIBC_FUNCTION(int, issignalingf, (float x)) { | ||
| - return fputil::issignaling(x); | ||
| + return fputil::issignaling_impl(x); | ||
| } | ||
|
|
||
| } // namespace LIBC_NAMESPACE_DECL | ||
| diff -ruN --strip-trailing-cr a/libc/src/math/generic/issignalingl.cpp b/libc/src/math/generic/issignalingl.cpp | ||
| --- a/libc/src/math/generic/issignalingl.cpp | ||
| +++ b/libc/src/math/generic/issignalingl.cpp | ||
| @@ -14,7 +14,7 @@ | ||
| namespace LIBC_NAMESPACE_DECL { | ||
|
|
||
| LLVM_LIBC_FUNCTION(int, issignalingl, (long double x)) { | ||
| - return fputil::issignaling(x); | ||
| + return fputil::issignaling_impl(x); | ||
| } | ||
|
|
||
| } // namespace LIBC_NAMESPACE_DECL | ||
| diff -ruN --strip-trailing-cr a/libc/src/__support/FPUtil/BasicOperations.h b/libc/src/__support/FPUtil/BasicOperations.h | ||
| --- a/libc/src/__support/FPUtil/BasicOperations.h | ||
| +++ b/libc/src/__support/FPUtil/BasicOperations.h | ||
| @@ -247,8 +247,9 @@ | ||
| return (x > y ? x - y : 0); | ||
| } | ||
|
|
||
| +// Avoid reusing `issignaling` macro. | ||
| template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0> | ||
| -LIBC_INLINE int issignaling(const T &x) { | ||
| +LIBC_INLINE int issignaling_impl(const T &x) { | ||
| FPBits<T> sx(x); | ||
| return sx.is_signaling_nan(); | ||
| } | ||
| diff -ruN --strip-trailing-cr a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp | ||
| --- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp | ||
| +++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp | ||
| @@ -61,6 +61,8 @@ | ||
| case Instruction::ICmp: | ||
| case VPInstruction::ActiveLaneMask: | ||
| return inferScalarType(R->getOperand(1)); | ||
| + case VPInstruction::ExplicitVectorLength: | ||
| + return Type::getIntNTy(Ctx, 32); | ||
| case VPInstruction::FirstOrderRecurrenceSplice: | ||
| case VPInstruction::Not: | ||
| return SetResultTyFromOp(); | ||
| diff -ruN --strip-trailing-cr a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h | ||
| --- a/llvm/lib/Transforms/Vectorize/VPlan.h | ||
| +++ b/llvm/lib/Transforms/Vectorize/VPlan.h | ||
| @@ -1649,6 +1649,16 @@ | ||
| MayWriteToMemory(CI.mayWriteToMemory()), | ||
| MayHaveSideEffects(CI.mayHaveSideEffects()) {} | ||
|
|
||
| + VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID, | ||
| + ArrayRef<VPValue *> CallArguments, Type *Ty, | ||
| + bool MayReadFromMemory, bool MayWriteToMemory, | ||
| + bool MayHaveSideEffects, DebugLoc DL = {}) | ||
| + : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments), | ||
| + VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty), | ||
| + MayReadFromMemory(MayReadFromMemory), | ||
| + MayWriteToMemory(MayWriteToMemory), | ||
| + MayHaveSideEffects(MayHaveSideEffects) {} | ||
| + | ||
| ~VPWidenIntrinsicRecipe() override = default; | ||
|
|
||
| VPWidenIntrinsicRecipe *clone() override { | ||
| @@ -1686,6 +1696,8 @@ | ||
| void print(raw_ostream &O, const Twine &Indent, | ||
| VPSlotTracker &SlotTracker) const override; | ||
| #endif | ||
| + | ||
| + bool onlyFirstLaneUsed(const VPValue *Op) const override; | ||
| }; | ||
|
|
||
| /// A recipe for widening Call instructions using library calls. | ||
| diff -ruN --strip-trailing-cr a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | ||
| --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | ||
| +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | ||
| @@ -79,7 +79,6 @@ | ||
| return !cast<VPWidenCallRecipe>(this) | ||
| ->getCalledScalarFunction() | ||
| ->onlyReadsMemory(); | ||
| - case VPWidenIntrinsicSC: | ||
| return cast<VPWidenIntrinsicRecipe>(this)->mayWriteToMemory(); | ||
| case VPBranchOnMaskSC: | ||
| case VPScalarIVStepsSC: | ||
| @@ -1042,6 +1041,14 @@ | ||
| return Intrinsic::getBaseName(VectorIntrinsicID); | ||
| } | ||
|
|
||
| +bool VPWidenIntrinsicRecipe::onlyFirstLaneUsed(const VPValue *Op) const { | ||
| + assert(is_contained(operands(), Op) && "Op must be an operand of the recipe"); | ||
| + // Vector predication intrinsics only demand the the first lane the last | ||
| + // operand (the EVL operand). | ||
| + return VPIntrinsic::isVPIntrinsic(VectorIntrinsicID) && | ||
| + Op == getOperand(getNumOperands() - 1); | ||
| +} | ||
| + | ||
| #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | ||
| void VPWidenIntrinsicRecipe::print(raw_ostream &O, const Twine &Indent, | ||
| VPSlotTracker &SlotTracker) const { | ||
| diff -ruN --strip-trailing-cr a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | ||
| --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | ||
| +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | ||
| @@ -1353,6 +1353,7 @@ | ||
| /// Replace recipes with their EVL variants. | ||
| static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { | ||
| SmallVector<VPValue *> HeaderMasks = collectAllHeaderMasks(Plan); | ||
| + VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType()); | ||
| for (VPValue *HeaderMask : collectAllHeaderMasks(Plan)) { | ||
| for (VPUser *U : collectUsersRecursively(HeaderMask)) { | ||
| auto *CurRecipe = dyn_cast<VPRecipeBase>(U); | ||
| @@ -1384,6 +1385,14 @@ | ||
| VPValue *NewMask = GetNewMask(Red->getCondOp()); | ||
| return new VPReductionEVLRecipe(*Red, EVL, NewMask); | ||
| }) | ||
| + .Case<VPWidenSelectRecipe>([&](VPWidenSelectRecipe *Sel) { | ||
| + SmallVector<VPValue *> Ops(Sel->operands()); | ||
| + Ops.push_back(&EVL); | ||
| + return new VPWidenIntrinsicRecipe(Intrinsic::vp_select, Ops, | ||
| + TypeInfo.inferScalarType(Sel), | ||
| + false, false, false); | ||
| + }) | ||
| + | ||
| .Default([&](VPRecipeBase *R) { return nullptr; }); | ||
|
|
||
| if (!NewRecipe) | ||
| @@ -1637,8 +1646,9 @@ | ||
| // zero. | ||
| assert(IG->getIndex(IRInsertPos) != 0 && | ||
| "index of insert position shouldn't be zero"); | ||
| + auto &DL = IRInsertPos->getDataLayout(); | ||
| APInt Offset(32, | ||
| - getLoadStoreType(IRInsertPos)->getScalarSizeInBits() / 8 * | ||
| + DL.getTypeAllocSize(getLoadStoreType(IRInsertPos)) * | ||
| IG->getIndex(IRInsertPos), | ||
| /*IsSigned=*/true); | ||
| VPValue *OffsetVPV = Plan.getOrAddLiveIn( | ||
| diff -ruN --strip-trailing-cr a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp | ||
| --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp | ||
| +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp | ||
| @@ -138,6 +138,10 @@ | ||
| }; | ||
| for (const VPUser *U : EVL.users()) { | ||
| if (!TypeSwitch<const VPUser *, bool>(U) | ||
| + .Case<VPWidenIntrinsicRecipe>( | ||
| + [&](const VPWidenIntrinsicRecipe *S) { | ||
| + return VerifyEVLUse(*S, S->getNumOperands() - 1); | ||
| + }) | ||
| .Case<VPWidenStoreEVLRecipe>([&](const VPWidenStoreEVLRecipe *S) { | ||
| return VerifyEVLUse(*S, 2); | ||
| }) | ||
| diff -ruN --strip-trailing-cr a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-different-insert-position.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-different-insert-position.ll | ||
| --- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-different-insert-position.ll | ||
| +++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-different-insert-position.ll | ||
| @@ -154,6 +154,92 @@ | ||
| exit: | ||
| ret void | ||
| } | ||
| + | ||
| +; FIXME: Currently the start address of the interleav group is computed | ||
| +; incorrectly. | ||
| +define i64 @interleave_group_load_pointer_type(ptr %start, ptr %end) { | ||
| +; CHECK-LABEL: define i64 @interleave_group_load_pointer_type( | ||
| +; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) { | ||
| +; CHECK-NEXT: [[ENTRY:.*]]: | ||
| +; CHECK-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64 | ||
| +; CHECK-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64 | ||
| +; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[END1]], [[START2]] | ||
| +; CHECK-NEXT: [[TMP1:%.*]] = udiv i64 [[TMP0]], 24 | ||
| +; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 | ||
| +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], 4 | ||
| +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | ||
| +; CHECK: [[VECTOR_PH]]: | ||
| +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4 | ||
| +; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 | ||
| +; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i64 4, i64 [[N_MOD_VF]] | ||
| +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP4]] | ||
| +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[N_VEC]], 24 | ||
| +; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP5]] | ||
| +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | ||
| +; CHECK: [[VECTOR_BODY]]: | ||
| +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | ||
| +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ] | ||
| +; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 24 | ||
| +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 0 | ||
| +; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP6]] | ||
| +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 16 | ||
| +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 -8 | ||
| +; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x ptr>, ptr [[TMP8]], align 8 | ||
| +; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x ptr> [[WIDE_VEC]], <12 x ptr> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9> | ||
| +; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <12 x ptr> [[WIDE_VEC]], <12 x ptr> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10> | ||
| +; CHECK-NEXT: [[TMP9:%.*]] = ptrtoint <4 x ptr> [[STRIDED_VEC3]] to <4 x i64> | ||
| +; CHECK-NEXT: [[TMP10:%.*]] = ptrtoint <4 x ptr> [[STRIDED_VEC]] to <4 x i64> | ||
| +; CHECK-NEXT: [[TMP11:%.*]] = or <4 x i64> [[TMP9]], [[TMP10]] | ||
| +; CHECK-NEXT: [[TMP12]] = or <4 x i64> [[TMP11]], [[VEC_PHI]] | ||
| +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | ||
| +; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | ||
| +; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] | ||
| +; CHECK: [[MIDDLE_BLOCK]]: | ||
| +; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP12]]) | ||
| +; CHECK-NEXT: br label %[[SCALAR_PH]] | ||
| +; CHECK: [[SCALAR_PH]]: | ||
| +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ] | ||
| +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] | ||
| +; CHECK-NEXT: br label %[[LOOP:.*]] | ||
| +; CHECK: [[LOOP]]: | ||
| +; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ] | ||
| +; CHECK-NEXT: [[RED:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ] | ||
| +; CHECK-NEXT: [[GEP_16:%.*]] = getelementptr i8, ptr [[PTR_IV]], i64 16 | ||
| +; CHECK-NEXT: [[L_16:%.*]] = load ptr, ptr [[GEP_16]], align 8 | ||
| +; CHECK-NEXT: [[P_16:%.*]] = ptrtoint ptr [[L_16]] to i64 | ||
| +; CHECK-NEXT: [[GEP_8:%.*]] = getelementptr i8, ptr [[PTR_IV]], i64 8 | ||
| +; CHECK-NEXT: [[L_8:%.*]] = load ptr, ptr [[GEP_8]], align 8 | ||
| +; CHECK-NEXT: [[P_8:%.*]] = ptrtoint ptr [[L_8]] to i64 | ||
| +; CHECK-NEXT: [[OR_1:%.*]] = or i64 [[P_16]], [[P_8]] | ||
| +; CHECK-NEXT: [[RED_NEXT]] = or i64 [[OR_1]], [[RED]] | ||
| +; CHECK-NEXT: [[PTR_IV_NEXT]] = getelementptr nusw i8, ptr [[PTR_IV]], i64 24 | ||
| +; CHECK-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV]], [[END]] | ||
| +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]] | ||
| +; CHECK: [[EXIT]]: | ||
| +; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i64 [ [[RED_NEXT]], %[[LOOP]] ] | ||
| +; CHECK-NEXT: ret i64 [[RED_NEXT_LCSSA]] | ||
| +; | ||
| +entry: | ||
| + br label %loop | ||
| + | ||
| +loop: | ||
| + %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop ] | ||
| + %red = phi i64 [ 0, %entry ], [ %red.next, %loop ] | ||
| + %gep.16 = getelementptr i8, ptr %ptr.iv, i64 16 | ||
| + %l.16 = load ptr, ptr %gep.16, align 8 | ||
| + %p.16 = ptrtoint ptr %l.16 to i64 | ||
| + %gep.8 = getelementptr i8, ptr %ptr.iv, i64 8 | ||
| + %l.8 = load ptr, ptr %gep.8, align 8 | ||
| + %p.8 = ptrtoint ptr %l.8 to i64 | ||
| + %or.1 = or i64 %p.16, %p.8 | ||
| + %red.next = or i64 %or.1, %red | ||
| + %ptr.iv.next = getelementptr nusw i8, ptr %ptr.iv, i64 24 | ||
| + %ec = icmp eq ptr %ptr.iv, %end | ||
| + br i1 %ec, label %exit, label %loop | ||
| + | ||
| +exit: | ||
| + ret i64 %red.next | ||
| +} | ||
| ;. | ||
| ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} | ||
| ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} | ||
| @@ -161,4 +247,6 @@ | ||
| ; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]} | ||
| ; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]} | ||
| ; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]} | ||
| +; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]} | ||
| +; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]} | ||
| ;. | ||
| diff -ruN --strip-trailing-cr a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll | ||
| --- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll | ||
| +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll | ||
| @@ -70,7 +70,7 @@ | ||
| ; IF-EVL-INLOOP-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP17]], i32 0 | ||
| ; IF-EVL-INLOOP-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP18]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP12]]) | ||
| ; IF-EVL-INLOOP-NEXT: [[TMP19:%.*]] = icmp sgt <vscale x 4 x i32> [[VP_OP_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 3, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer) | ||
| -; IF-EVL-INLOOP-NEXT: [[TMP20:%.*]] = select <vscale x 4 x i1> [[TMP19]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer | ||
| +; IF-EVL-INLOOP-NEXT: [[TMP20:%.*]] = call <vscale x 4 x i32> @llvm.vp.select.nxv4i32(<vscale x 4 x i1> [[TMP19]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP12]]) | ||
| ; IF-EVL-INLOOP-NEXT: [[TMP21:%.*]] = call i32 @llvm.vp.reduce.add.nxv4i32(i32 0, <vscale x 4 x i32> [[TMP20]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), i32 [[TMP12]]) | ||
| ; IF-EVL-INLOOP-NEXT: [[TMP22]] = add i32 [[TMP21]], [[VEC_PHI]] | ||
| ; IF-EVL-INLOOP-NEXT: [[TMP23:%.*]] = zext i32 [[TMP12]] to i64 | ||
| diff -ruN --strip-trailing-cr a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll | ||
| --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll | ||
| +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll | ||
| @@ -0,0 +1,65 @@ | ||
| +; REQUIRES: asserts | ||
| + | ||
| + ; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \ | ||
| + ; RUN: -force-tail-folding-style=data-with-evl \ | ||
| + ; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \ | ||
| + ; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefix=IF-EVL %s | ||
| + | ||
| + define void @vp_select(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) { | ||
| + ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { | ||
| + ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF | ||
| + ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count | ||
| + ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count | ||
| + | ||
| + ; IF-EVL: vector.ph: | ||
| + ; IF-EVL-NEXT: Successor(s): vector loop | ||
| + | ||
| + ; IF-EVL: <x1> vector loop: { | ||
| + ; IF-EVL-NEXT: vector.body: | ||
| + ; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION | ||
| + ; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEX:%[0-9]+]]> | ||
| + ; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]> | ||
| + ; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]> | ||
| + ; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1> | ||
| + ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]> | ||
| + ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> | ||
| + ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> | ||
| + ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%c>, vp<[[ST]]> | ||
| + ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> | ||
| + ; IF-EVL-NEXT: WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]> | ||
| + ; IF-EVL-NEXT: WIDEN ir<[[CMP:%.+]]> = icmp sgt ir<[[LD1]]>, ir<[[LD2]]> | ||
| + ; IF-EVL-NEXT: WIDEN ir<[[SUB:%.+]]> = vp.sub ir<0>, ir<[[LD2]]>, vp<[[EVL]]> | ||
| + ; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SELECT:%.+]]> = call llvm.vp.select(ir<[[CMP]]>, ir<%1>, ir<%2>, vp<[[EVL]]>) | ||
| + ; IF-EVL-NEXT: WIDEN ir<[[ADD:%.+]]> = vp.add vp<[[SELECT]]>, ir<[[LD1]]>, vp<[[EVL]]> | ||
| + ; IF-EVL-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> | ||
| + ; IF-EVL-NEXT: vp<[[PTR3:%.+]]> = vector-pointer ir<[[GEP3]]> | ||
| + ; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<[[EVL]]> | ||
| + ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 | ||
| + ; IF-EVL-NEXT: EMIT vp<[[IV_NEX]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> | ||
| + ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%[0-9]+]]> = add vp<[[IV]]>, vp<[[VFUF]]> | ||
| + ; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]> | ||
| + ; IF-EVL-NEXT: No successors | ||
| + ; IF-EVL-NEXT: } | ||
| + | ||
| + entry: | ||
| + br label %for.body | ||
| + | ||
| + for.body: | ||
| + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] | ||
| + %arrayidx = getelementptr inbounds i32, ptr %b, i64 %indvars.iv | ||
| + %0 = load i32, ptr %arrayidx, align 4 | ||
| + %arrayidx3 = getelementptr inbounds i32, ptr %c, i64 %indvars.iv | ||
| + %1 = load i32, ptr %arrayidx3, align 4 | ||
| + %cmp4 = icmp sgt i32 %0, %1 | ||
| + %2 = sub i32 0, %1 | ||
| + %cond.p = select i1 %cmp4, i32 %1, i32 %2 | ||
| + %cond = add i32 %cond.p, %0 | ||
| + %arrayidx15 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv | ||
| + store i32 %cond, ptr %arrayidx15, align 4 | ||
| + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 | ||
| + %exitcond.not = icmp eq i64 %indvars.iv.next, %N | ||
| + br i1 %exitcond.not, label %exit, label %for.body | ||
| + | ||
| + exit: | ||
| + ret void | ||
| + } |
Oops, something went wrong.