Improve i64 emulation optimization for add and sub for VC

When the i64 source values are created as low/high parts combination,
the compiler can eliminate the wrregion and bitcast operations and
propagate the low/high parts directly to the i64 emulation routines.

Also, when the i64 source value of an add or sub instruction is a
zero-extended i32 value, the compiler can optimize the high part
addition or substraction.

Author: vmustya

IGC/VectorCompiler/lib/GenXCodeGen/GenXConstants.cpp

@@ -1437,7 +1437,7 @@ Instruction *ConstantLoader::load(Instruction *InsertBefore) {
 
   if (!PackedFloat && !PackedIntScale &&
       !isa<UndefValue>(C)) { // not packed int constant or undef
-    if (auto CC = getConsolidatedConstant(C)) {
+    if (auto *CC = getConsolidatedConstant(C)) {
       // We're loading a vector of byte or short (but not i1). Use int so the
       // instruction does not use so many channels. This may also save it being
       // split by legalization.
@@ -1477,6 +1477,10 @@ Instruction *ConstantLoader::load(Instruction *InsertBefore) {
 
   NewInst = cast<Instruction>(Builder.CreateBitCast(NewInst, Ty));
 
+  if (Ty->isIntOrIntVectorTy(64) && !allowI64Ops())
+    NewInst =
+        genx::emulateI64Operation(&Subtarget, NewInst, EmulationFlag::RAUWE);
+
   if (AddedInstructions)
     AddedInstructions->push_back(NewInst);
 
@@ -1548,9 +1552,11 @@ Instruction *ConstantLoader::loadPackedInt(Instruction *Inst) {
       Val -= PackedIntAdjust;
       Val /= PackedIntScale;
     }
-    PackedVals.push_back(ConstantInt::get(PackTy, Val, /*isSigned=*/true));
-    IGC_ASSERT(cast<ConstantInt>(PackedVals.back())->getSExtValue() >= -8 &&
-               cast<ConstantInt>(PackedVals.back())->getSExtValue() <= 15);
+    auto *PackedConst =
+        cast<ConstantInt>(ConstantInt::get(PackTy, Val, /*isSigned=*/true));
+    PackedVals.push_back(PackedConst);
+    IGC_ASSERT(PackedConst->getSExtValue() >= -8 &&
+               PackedConst->getSExtValue() <= 15);
   }
 
   ConstantLoader Packed(ConstantVector::get(PackedVals), Subtarget, DL);

IGC/VectorCompiler/lib/GenXCodeGen/GenXEmulate.cpp

@@ -175,8 +175,10 @@ class GenXEmulate : public ModulePass {
     // handles genx_{XX}add_sat cases
     Value *visitGenxAddSat(CallInst &CI);
 
+    // handles genx_constanti
+    Value *visitConstantI(CallInst &CI);
+
     // [+] bitcast
-    // [-] genx.constanti ?
     // [-] genx.scatter ?
     // [-] genx.gather ?
     Value *visitCallInst(CallInst &CI);
@@ -191,6 +193,7 @@ class GenXEmulate : public ModulePass {
     static bool isI64ToFP(const Instruction &I);
     static bool isI64Cmp(const Instruction &I);
     static bool isI64AddSat(const Instruction &I);
+    static bool isI64Constant(const Instruction &I);
     static Value *detectBitwiseNot(BinaryOperator &);
     static Type *changeScalarType(Type *T, Type *NewTy);
 
@@ -216,7 +219,7 @@ class GenXEmulate : public ModulePass {
     bool needsEmulation() const {
       return (SplitBuilder.IsI64Operation() || isI64Cmp(Inst) ||
               isConvertOfI64(Inst) || isI64PointerOp(Inst) ||
-              isI64AddSat(Inst));
+              isI64AddSat(Inst) || isI64Constant(Inst));
     }
 
     IRBuilder getIRBuilder() {
@@ -454,6 +457,13 @@ bool GenXEmulate::Emu64Expander::isI64AddSat(const Instruction &I) {
   }
   return false;
 }
+bool GenXEmulate::Emu64Expander::isI64Constant(const Instruction &I) {
+  auto IID = vc::getAnyIntrinsicID(&I);
+  if (IID != GenXIntrinsic::genx_constanti)
+    return false;
+  Type *RetTy = I.getType();
+  return RetTy->isIntOrIntVectorTy(64);
+}
 
 Value *GenXEmulate::Emu64Expander::detectBitwiseNot(BinaryOperator &Op) {
   if (Instruction::Xor != Op.getOpcode())
@@ -564,8 +574,24 @@ Value *GenXEmulate::Emu64Expander::visitAdd(BinaryOperator &Op) {
   auto AddcRes = buildAddc(Inst.getModule(), Builder, *Src0.Lo, *Src1.Lo,
                            "int_emu.add64.lo.");
   auto *AddLo = AddcRes.Val;
-  auto *AddHi =
-      buildTernaryAddition(Builder, *AddcRes.CB, *Src0.Hi, *Src1.Hi, "add_hi");
+  Value *AddHi = nullptr;
+
+  auto *Src0HiConst = dyn_cast<Constant>(Src0.Hi);
+  auto *Src1HiConst = dyn_cast<Constant>(Src1.Hi);
+  if (Src0HiConst && Src1HiConst && Src0HiConst->isZeroValue() &&
+      Src1HiConst->isZeroValue()) {
+    // Optimization for the case when both high parts are zero
+    AddHi = AddcRes.CB;
+  } else if (Src0HiConst && Src0HiConst->isZeroValue()) {
+    // Optimization for the case when src0 high part is zero
+    AddHi = Builder.CreateAdd(AddcRes.CB, Src1.Hi, "add_hi");
+  } else if (Src1HiConst && Src1HiConst->isZeroValue()) {
+    // Optimization for the case when src1 high part is zero
+    AddHi = Builder.CreateAdd(AddcRes.CB, Src0.Hi, "add_hi");
+  } else {
+    AddHi = buildTernaryAddition(Builder, *AddcRes.CB, *Src0.Hi, *Src1.Hi,
+                                 "add_hi");
+  }
   return SplitBuilder.combineLoHiSplit(
       {AddLo, AddHi}, Twine("int_emu.") + Op.getOpcodeName() + ".",
       Inst.getType()->isIntegerTy());
@@ -589,9 +615,26 @@ Value *GenXEmulate::Emu64Expander::visitSub(BinaryOperator &Op) {
   auto *Borrow =
       Builder.CreateExtractValue(SubbVal, {IdxSubb_Borrow}, "subb.borrow");
   auto *MinusBorrow = Builder.CreateNeg(Borrow, "borrow.negate");
-  auto *MinusS1Hi = Builder.CreateNeg(Src1.Hi, "negative.src1_hi");
-  auto *SubHi = buildTernaryAddition(Builder, *Src0.Hi, *MinusBorrow,
-                                     *MinusS1Hi, "sub_hi");
+
+  Value *SubHi = nullptr;
+
+  auto *Src0HiConst = dyn_cast<Constant>(Src0.Hi);
+  auto *Src1HiConst = dyn_cast<Constant>(Src1.Hi);
+  if (Src0HiConst && Src1HiConst && Src0HiConst->isZeroValue() &&
+      Src1HiConst->isZeroValue()) {
+    // Optimization for the case when both high parts are zero
+    SubHi = MinusBorrow;
+  } else if (Src0HiConst && Src0HiConst->isZeroValue()) {
+    // Optimization for the case when src0 high part is zero
+    SubHi = Builder.CreateSub(MinusBorrow, Src1.Hi, "sub_hi");
+  } else if (Src1HiConst && Src1HiConst->isZeroValue()) {
+    // Optimization for the case when src1 high part is zero
+    SubHi = Builder.CreateAdd(Src0.Hi, MinusBorrow, "sub_hi");
+  } else {
+    auto *MinusS1Hi = Builder.CreateNeg(Src1.Hi, "negative.src1_hi");
+    SubHi = buildTernaryAddition(Builder, *Src0.Hi, *MinusBorrow, *MinusS1Hi,
+                                 "sub_hi");
+  }
   return SplitBuilder.combineLoHiSplit(
       {SubLo, SubHi}, Twine("int_emu.") + Op.getOpcodeName() + ".",
       Inst.getType()->isIntegerTy());
@@ -1206,6 +1249,57 @@ Value *GenXEmulate::Emu64Expander::visitGenxAddSat(CallInst &CI) {
   return Result;
 }
 
+Value *GenXEmulate::Emu64Expander::visitConstantI(CallInst &CI) {
+  auto Builder = getIRBuilder();
+  auto *M = CI.getModule();
+  auto &Ctx = M->getContext();
+
+  auto *Src = CI.getOperand(0);
+  auto *SrcTy = cast<IGCLLVM::FixedVectorType>(Src->getType());
+  auto NumElements = SrcTy->getNumElements();
+
+  SmallVector<uint32_t, 8> Values;
+  bool CanBeZext = false;
+
+  if (auto *SrcC = dyn_cast<ConstantDataVector>(Src)) {
+    CanBeZext = true;
+    for (unsigned I = 0; CanBeZext && I < NumElements; I++) {
+      auto Val = SrcC->getElementAsInteger(I);
+      CanBeZext &= Val <= std::numeric_limits<uint32_t>::max();
+      Values.emplace_back(static_cast<uint32_t>(Val));
+    }
+  }
+
+  Value *Result = nullptr;
+  if (CanBeZext) {
+    // Can be represented as zext from 32-bit values, so create 32-bit constants
+    // and zext it to 64-bit.
+    auto *NewSrc = ConstantDataVector::get(Ctx, Values);
+    auto *NewTy = NewSrc->getType();
+    IGC_ASSERT_EXIT(NewTy->getScalarSizeInBits() == 32);
+
+    auto *NewIntr =
+        vc::getAnyDeclaration(M, GenXIntrinsic::genx_constanti, {NewTy});
+    auto *NewConst = Builder.CreateCall(NewIntr, {NewSrc});
+    Result = Builder.CreateZExt(NewConst, CI.getType());
+  } else {
+    // Cannot be represented as zext from 32-bit values, so bitcast the
+    // <N x i64> constant to <2N x i32>.
+    auto *NewTy =
+        IGCLLVM::FixedVectorType::get(Builder.getInt32Ty(), NumElements * 2);
+
+    auto *Cast = Builder.CreateBitCast(Src, NewTy);
+    IGC_ASSERT_EXIT(isa<Constant>(Cast));
+
+    auto *NewIntr =
+        vc::getAnyDeclaration(M, GenXIntrinsic::genx_constanti, {NewTy});
+    auto *NewConst = Builder.CreateCall(NewIntr, {Cast});
+    Result = Builder.CreateBitCast(NewConst, CI.getType());
+  }
+
+  return Result;
+}
+
 Value *GenXEmulate::Emu64Expander::visitCallInst(CallInst &CI) {
   switch (vc::getAnyIntrinsicID(&Inst)) {
   case Intrinsic::smax:
@@ -1225,6 +1319,8 @@ Value *GenXEmulate::Emu64Expander::visitCallInst(CallInst &CI) {
   case GenXIntrinsic::genx_uuadd_sat:
   case GenXIntrinsic::genx_ssadd_sat:
     return visitGenxAddSat(CI);
+  case GenXIntrinsic::genx_constanti:
+    return visitConstantI(CI);
   }
   return nullptr;
 }

IGC/VectorCompiler/lib/GenXCodeGen/GenXUtil.cpp

@@ -23,6 +23,7 @@ SPDX-License-Identifier: MIT
 #include "vc/Utils/GenX/InternalMetadata.h"
 #include "vc/Utils/GenX/PredefinedVariable.h"
 #include "vc/Utils/GenX/Printf.h"
+#include "vc/Utils/GenX/Region.h"
 #include "vc/Utils/General/Types.h"
 
 #include "llvmWrapper/IR/Instructions.h"
@@ -1240,7 +1241,97 @@ IVSplitter::HalfSplit IVSplitter::splitOperandHalf(unsigned SourceIdx,
   return splitValueHalf(*Inst.getOperand(SourceIdx), FoldConstants);
 }
 
+// Returns tuple of InsertTo, InsertV, Offset if Src is a legalized
+// <2;1,0> write-region with full mask, otherwise nullptrs and 0.
+static std::tuple<Value *, Value *, unsigned> getInsertedValue(Value *Src) {
+  auto *Ty = cast<IGCLLVM::FixedVectorType>(Src->getType());
+  auto NumElements = Ty->getNumElements();
+  IGC_ASSERT(Ty->getElementType()->isIntegerTy(32));
+
+  auto *I = dyn_cast<CallInst>(Src);
+  if (!I || !GenXIntrinsic::isWrRegion(I))
+    return {nullptr, nullptr, 0};
+
+  vc::CMRegion Wr(I);
+  // Only direct, full-mask regions are supported.
+  if (Wr.Indirect || Wr.Mask)
+    return {nullptr, nullptr, 0};
+
+  // Support only 1D regions with stride == 2
+  if (!Wr.is1D() || Wr.Stride != 2 || Wr.Width != NumElements / 2)
+    return {nullptr, nullptr, 0};
+
+  auto *InsertTo = I->getArgOperand(0);
+  auto *InsertV = I->getArgOperand(1);
+  unsigned Offset = Wr.Offset / 4; // in elements
+
+  auto *InsertTy = InsertV->getType();
+  if (InsertTy->isIntegerTy(32)) {
+    IRBuilder<> Builder(I);
+    auto *VTy = IGCLLVM::FixedVectorType::get(InsertTy, 1);
+    InsertV = Builder.CreateBitCast(InsertV, VTy);
+  }
+
+  return {InsertTo, InsertV, Offset};
+}
+
+static IVSplitter::LoHiSplit matchLoHiCombine(Value &V) {
+  auto *Cast = dyn_cast<BitCastInst>(&V);
+  if (!Cast)
+    return {nullptr, nullptr};
+
+  auto *SrcTy = Cast->getSrcTy();
+  if (!SrcTy->isIntOrIntVectorTy(32))
+    return {nullptr, nullptr};
+
+  Value *Lo = nullptr;
+  Value *Hi = nullptr;
+
+  // Check the first write-region
+  auto [InsertTo, InsertV, Offset] = getInsertedValue(Cast->getOperand(0));
+  if (!InsertTo || !InsertV || (Offset != 1 && Offset != 0))
+    return {nullptr, nullptr};
+
+  if (Offset == 0)
+    Lo = InsertV;
+  else
+    Hi = InsertV;
+
+  // Check the second write-region
+  std::tie(InsertTo, InsertV, Offset) = getInsertedValue(InsertTo);
+  if (!InsertTo || !InsertV || (Offset != 1 && Offset != 0))
+    return {nullptr, nullptr};
+
+  if (Offset == 0)
+    Lo = InsertV;
+  else
+    Hi = InsertV;
+
+  return {Lo, Hi};
+}
+
 IVSplitter::LoHiSplit IVSplitter::splitValueLoHi(Value &V, bool FoldConstants) {
+  // When the source is a zext from i32 to i64, we can just take the source
+  // and a zero constant as the lo/hi parts.
+  if (auto *ZExt = dyn_cast<ZExtInst>(&V)) {
+    auto *Src = ZExt->getOperand(0);
+    auto *SrcTy = Src->getType();
+    if (SrcTy->isIntegerTy(32)) {
+      IRBuilder<> Builder(ZExt);
+      auto *VTy = IGCLLVM::FixedVectorType::get(SrcTy, 1);
+      auto *Cast = Builder.CreateBitCast(Src, VTy);
+      return {Cast, Constant::getNullValue(VTy)};
+    }
+    if (SrcTy->isIntOrIntVectorTy(32)) {
+      // special case: splitting a zext of i32 to i64
+      return {Src, Constant::getNullValue(SrcTy)};
+    }
+  }
+
+  auto MaybeSplit = matchLoHiCombine(V);
+  if (MaybeSplit.Lo && MaybeSplit.Hi)
+    return MaybeSplit;
+
   auto Splitted = splitValue(V, RegionType::LoRegion, ".LoSplit",
                              RegionType::HiRegion, ".HiSplit", FoldConstants);
   return {Splitted.first, Splitted.second};

IGC/VectorCompiler/test/Emulation/emu_i64_add64.ll

@@ -101,8 +101,38 @@ define dllexport spir_kernel void @test_kernel_constant(<8 x i64> %left) {
   ret void
 }
 
+; COM: ===============================
+; COM:             TEST #4
+; COM: ===============================
+; COM: add64 with zero-extended src1 operand:
+; COM: 1. operands are splitted to lo/hi parts
+; COM: 2. [add_lo, carry] = genx_addc(src0.l0, src1.lo)
+; COM: 3. add_hi = add carry, src0.hi
+; COM: 4. add64  = combine(add_lo,add_hi)
+
+; CHECK-LABEL: @test_kernel_zext
+
+; CHECK: [[IV1:%[^ ]+.iv32cast[0-9]*]] = bitcast <[[OT:8 x i64]]> %left to <[[CT:16 x i32]]>
+; CHECK-NEXT: [[Lo_l:%[^ ]+.LoSplit[0-9]*]] = call <[[ET:8 x i32]]> [[rgn:@llvm.genx.rdregioni.[^(]+]](<[[CT]]> [[IV1]], [[low_reg:i32 0, i32 8, i32 2, i16 0,]]
+; CHECK-NEXT: [[Hi_l:%[^ ]+.HiSplit[0-9]*]] = call <[[ET]]> [[rgn]](<[[CT]]> [[IV1]], [[high_reg:i32 0, i32 8, i32 2, i16 4,]]
+
+; CHECK: [[ADDC:%[^ ]+]] = call { <[[ET]]>, <[[ET]]> } @llvm.genx.addc.{{[^(]+}}(<[[ET]]> [[Lo_l]], <[[ET]]> %right_i32)
+; CHECK-NEXT: [[ADDC_ADD:%[^ ]+]] = extractvalue { <[[ET]]>, <[[ET]]> } [[ADDC]], 1
+; CHECK-NEXT: [[ADDC_CARRY:%[^ ]+]] = extractvalue { <[[ET]]>, <[[ET]]> } [[ADDC]], 0
+; CHECK-NEXT: [[Add_Hi:%add_hi]] = add <[[ET]]> [[ADDC_CARRY]], [[Hi_l]]
+
+define dllexport spir_kernel void @test_kernel_zext(i32 %0, i32 %1, i32 %2) {
+  %left = tail call <8 x i64> @llvm.genx.oword.ld.v8i64(i32 0, i32 %0, i32 0)
+  %right_i32 = tail call <8 x i32> @llvm.genx.oword.ld.v8i32(i32 0, i32 %1, i32 0)
+  %right = zext <8 x i32> %right_i32 to <8 x i64>
+  %add64 = add <8 x i64> %left, %right
+  tail call void @llvm.genx.oword.st.v8i64(i32 %2, i32 0, <8 x i64> %add64)
+  ret void
+}
+
 declare i64 @llvm.genx.rdregioni.i64.v8i64.i16(<8 x i64>, i32, i32, i32, i16, i32)
 declare <8 x i32> @llvm.genx.add3.v8i32.v8i32(<8 x i32>, <8 x i32>, <8 x i32>)
 declare { <8 x i32>, <8 x i32> } @llvm.genx.addc.v8i32.v8i32(<8 x i32>, <8 x i32>)
 declare <8 x i64> @llvm.genx.oword.ld.v8i64(i32, i32, i32)
+declare <8 x i32> @llvm.genx.oword.ld.v8i32(i32, i32, i32)
 declare void @llvm.genx.oword.st.v8i64(i32, i32, <8 x i64>)

IGC/VectorCompiler/test/Emulation/emu_i64_genx_absi.ll

@@ -29,20 +29,11 @@ declare <2 x i64> @llvm.genx.absi.v2i64(<2 x i64>)
 ; CHECK-NEXT: [[SUBB_SUB:%[^ ]+]] = extractvalue { <[[ET]]>, <[[ET]]> } [[SUBB]], 1
 ; CHECK-NEXT: [[SUBB_BORROW:%[^ ]+]] = extractvalue { <[[ET]]>, <[[ET]]> } [[SUBB]], 0
 ; CHECK-NEXT: [[BORROW_NEGATE:%[^ ]+]] = sub <[[ET]]> zeroinitializer, [[SUBB_BORROW]]
-; CHECK-NEXT: [[Hi_r_negate:%[^ ]+]] = sub <[[ET]]> zeroinitializer, [[Sub_Hi_r]]
-; CHECK-NEXT: [[Sub_Hi_Part:%[^ ]+]] = add <[[ET]]> zeroinitializer, [[BORROW_NEGATE]]
-; CHECK-NEXT: [[Sub_Hi:%[^ ]+]] = add <[[ET]]> [[Sub_Hi_Part]], [[Hi_r_negate]]
+; CHECK-NEXT: [[Sub_Hi:%[^ ]+]] = sub <[[ET]]> [[BORROW_NEGATE]], [[Sub_Hi_r]]
 
-; CHECK-NEXT: [[P_JOIN:%[^ ]+]] = call <[[CT]]> @llvm.genx.wrregioni.{{[^(]+}}(<[[CT]]> undef, <[[ET]]> [[SUBB_SUB]], [[low_reg]]
-; CHECK-NEXT: [[JOINED:%[^ ]+]] = call <[[CT]]> @llvm.genx.wrregioni.{{[^(]+}}(<[[CT]]> [[P_JOIN]], <[[ET]]> [[Sub_Hi]], [[high_reg]]
-; CHECK-NEXT: [[RECAST:%[^ ]+]] = bitcast <[[CT]]> [[JOINED]] to <[[RT]]>
-; CHECK-NEXT: [[RECAST2:%[^ ]+]] = bitcast <[[RT]]> [[RECAST]] to <[[CT]]>
-
-; CHECK-NEXT: [[NegLo:%[^ ]+]] = call <[[ET]]> [[rgn]](<[[CT]]> [[RECAST2]], [[low_reg]]
-; CHECK-NEXT: [[NegHi:%[^ ]+]] = call <[[ET]]> [[rgn]](<[[CT]]> [[RECAST2]], [[high_reg]]
-; CHECK-NEXT: [[CMP:%[^ ]+]] = icmp slt <[[ET]]> [[Hi_l]], zeroinitializer
-; CHECK-NEXT: [[SEL_LO:%[^ ]+]] = select <2 x i1> [[CMP]], <[[ET]]> [[NegLo]], <[[ET]]> [[Lo_l]]
-; CHECK-NEXT: [[SEL_HI:%[^ ]+]] = select <2 x i1> [[CMP]], <[[ET]]> [[NegHi]], <[[ET]]> [[Hi_l]]
+; CHECK: [[CMP:%[^ ]+]] = icmp slt <[[ET]]> [[Hi_l]], zeroinitializer
+; CHECK-NEXT: [[SEL_LO:%[^ ]+]] = select <2 x i1> [[CMP]], <[[ET]]> [[SUBB_SUB]], <[[ET]]> [[Lo_l]]
+; CHECK-NEXT: [[SEL_HI:%[^ ]+]] = select <2 x i1> [[CMP]], <[[ET]]> [[Sub_Hi]], <[[ET]]> [[Hi_l]]
 
 ; CHECK-NEXT: [[P_JOIN:%[^ ]+]] = call <[[CT]]> @llvm.genx.wrregioni.{{[^(]+}}(<[[CT]]> undef, <[[ET]]> [[SEL_LO]], [[low_reg]]
 ; CHECK-NEXT: [[JOINED:%[^ ]+]] = call <[[CT]]> @llvm.genx.wrregioni.{{[^(]+}}(<[[CT]]> [[P_JOIN]], <[[ET]]> [[SEL_HI]], [[high_reg]]

IGC/VectorCompiler/test/Emulation/emu_i64_genx_ssaddsat.ll

@@ -157,7 +157,6 @@ define dllexport spir_kernel void @test_ssaddsat_si64(i64 %lsop, i64 %rsop) {
 ; CHECK-NEXT: [[JOINED:%[^ ]+]] = call <[[CT]]> @llvm.genx.wrregioni.{{[^(]+}}(<[[CT]]> [[P_JOIN]], <[[ET]]> [[Hi]], [[high_reg]]
 ; CHECK-NEXT: [[RECAST:%[^ ]+]] = bitcast <[[CT]]> [[JOINED]] to <[[RT]]>
 ; COM: here emulation of i64->i16 sstrunc.sat, we don't have rigous checks for these
-; CHECK-NEXT: [[CAST_PREPARE:%[^ ]+]] = bitcast <[[RT]]> [[RECAST]] to <[[CT]]>
 ; COM: ...
 ; CHECK: [[SATURATE_I32:%[^ ]+]] = call <2 x i16> @llvm.genx.sstrunc.sat.v2i16.v2i32(<2 x i32> {{[^)]+}})
 ; CHECK-NEXT: [[USER:%[^ ]+]] = bitcast <2 x i16> [[SATURATE_I32]] to <2 x i16>