From 6d7e51de5ec46c1fcc7a7e80135f561a88a1296b Mon Sep 17 00:00:00 2001 From: Stanislav Mekhanoshin Date: Tue, 5 Nov 2024 13:59:14 -0800 Subject: [PATCH] [AMDGPU] Extend type support for update_dpp intrinsic (#114597) We can split 64-bit DPP as a post-RA pseudo if control values are supported, but cannot handle other types. --- .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 47 +- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 48 +- .../GlobalISel/llvm.amdgcn.update.dpp.ll | 40 +- .../atomic_optimizations_global_pointer.ll | 1756 +++--- .../atomic_optimizations_local_pointer.ll | 5303 ++++++++--------- llvm/test/CodeGen/AMDGPU/dpp64_combine.ll | 16 +- .../AMDGPU/global_atomics_scan_fadd.ll | 1457 +++-- .../AMDGPU/global_atomics_scan_fmax.ll | 926 ++- .../AMDGPU/global_atomics_scan_fmin.ll | 926 ++- .../AMDGPU/global_atomics_scan_fsub.ll | 1457 +++-- .../AMDGPU/llvm.amdgcn.update.dpp.gfx90a.ll | 276 + .../CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll | 30 +- 12 files changed, 6178 insertions(+), 6104 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.gfx90a.ll diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index ffff34417232bb..2fe6c4b965b95d 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -5495,6 +5495,13 @@ bool AMDGPULegalizerInfo::legalizeLaneOp(LegalizerHelper &Helper, } case Intrinsic::amdgcn_mov_dpp8: return LaneOp.addImm(MI.getOperand(3).getImm()).getReg(0); + case Intrinsic::amdgcn_update_dpp: + return LaneOp.addUse(Src1) + .addImm(MI.getOperand(4).getImm()) + .addImm(MI.getOperand(5).getImm()) + .addImm(MI.getOperand(6).getImm()) + .addImm(MI.getOperand(7).getImm()) + .getReg(0); default: llvm_unreachable("unhandled lane op"); } @@ -5504,7 +5511,7 @@ bool AMDGPULegalizerInfo::legalizeLaneOp(LegalizerHelper &Helper, Register Src0 = MI.getOperand(2).getReg(); Register Src1, Src2; if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane || - IsSetInactive || IsPermLane16) { + IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) { Src1 = MI.getOperand(3).getReg(); if (IID == Intrinsic::amdgcn_writelane || IsPermLane16) { Src2 = MI.getOperand(4).getReg(); @@ -5514,7 +5521,13 @@ bool AMDGPULegalizerInfo::legalizeLaneOp(LegalizerHelper &Helper, LLT Ty = MRI.getType(DstReg); unsigned Size = Ty.getSizeInBits(); - if (Size == 32) { + unsigned SplitSize = 32; + if (IID == Intrinsic::amdgcn_update_dpp && (Size % 64 == 0) && + ST.hasDPALU_DPP() && + AMDGPU::isLegalDPALU_DPPControl(MI.getOperand(4).getImm())) + SplitSize = 64; + + if (Size == SplitSize) { // Already legal return true; } @@ -5522,7 +5535,7 @@ bool AMDGPULegalizerInfo::legalizeLaneOp(LegalizerHelper &Helper, if (Size < 32) { Src0 = B.buildAnyExt(S32, Src0).getReg(0); - if (IsSetInactive || IsPermLane16) + if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) Src1 = B.buildAnyExt(LLT::scalar(32), Src1).getReg(0); if (IID == Intrinsic::amdgcn_writelane) @@ -5534,31 +5547,28 @@ bool AMDGPULegalizerInfo::legalizeLaneOp(LegalizerHelper &Helper, return true; } - if (Size % 32 != 0) + if (Size % SplitSize != 0) return false; - LLT PartialResTy = S32; + LLT PartialResTy = LLT::scalar(SplitSize); if (Ty.isVector()) { LLT EltTy = Ty.getElementType(); - switch (EltTy.getSizeInBits()) { - case 16: - PartialResTy = Ty.changeElementCount(ElementCount::getFixed(2)); - break; - case 32: + unsigned EltSize = EltTy.getSizeInBits(); + if (EltSize == SplitSize) { PartialResTy = EltTy; - break; - default: - // Handle all other cases via S32 pieces; - break; + } else if (EltSize == 16 || EltSize == 32) { + unsigned NElem = SplitSize / EltSize; + PartialResTy = Ty.changeElementCount(ElementCount::getFixed(NElem)); } + // Handle all other cases via S32/S64 pieces; } - SmallVector PartialRes; - unsigned NumParts = Size / 32; + SmallVector PartialRes; + unsigned NumParts = Size / SplitSize; MachineInstrBuilder Src0Parts = B.buildUnmerge(PartialResTy, Src0); MachineInstrBuilder Src1Parts, Src2Parts; - if (IsSetInactive || IsPermLane16) + if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) Src1Parts = B.buildUnmerge(PartialResTy, Src1); if (IID == Intrinsic::amdgcn_writelane) @@ -5567,7 +5577,7 @@ bool AMDGPULegalizerInfo::legalizeLaneOp(LegalizerHelper &Helper, for (unsigned i = 0; i < NumParts; ++i) { Src0 = Src0Parts.getReg(i); - if (IsSetInactive || IsPermLane16) + if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) Src1 = Src1Parts.getReg(i); if (IID == Intrinsic::amdgcn_writelane) @@ -7555,6 +7565,7 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, case Intrinsic::amdgcn_set_inactive: case Intrinsic::amdgcn_set_inactive_chain_arg: case Intrinsic::amdgcn_mov_dpp8: + case Intrinsic::amdgcn_update_dpp: return legalizeLaneOp(Helper, MI, IntrID); case Intrinsic::amdgcn_s_buffer_prefetch_data: return legalizeSBufferPrefetch(Helper, MI); diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 68ea6f622feca5..ae47a89864fbb8 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -6162,6 +6162,12 @@ static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N, IID == Intrinsic::amdgcn_set_inactive_chain_arg; SDLoc SL(N); MVT IntVT = MVT::getIntegerVT(ValSize); + const GCNSubtarget *ST = TLI.getSubtarget(); + unsigned SplitSize = 32; + if (IID == Intrinsic::amdgcn_update_dpp && (ValSize % 64 == 0) && + ST->hasDPALU_DPP() && + AMDGPU::isLegalDPALU_DPPControl(N->getConstantOperandVal(3))) + SplitSize = 64; auto createLaneOp = [&DAG, &SL, N, IID](SDValue Src0, SDValue Src1, SDValue Src2, MVT ValT) -> SDValue { @@ -6169,6 +6175,7 @@ static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N, switch (IID) { case Intrinsic::amdgcn_permlane16: case Intrinsic::amdgcn_permlanex16: + case Intrinsic::amdgcn_update_dpp: Operands.push_back(N->getOperand(6)); Operands.push_back(N->getOperand(5)); Operands.push_back(N->getOperand(4)); @@ -6206,13 +6213,15 @@ static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N, SDValue Src0 = N->getOperand(1); SDValue Src1, Src2; if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane || - IID == Intrinsic::amdgcn_mov_dpp8 || IsSetInactive || IsPermLane16) { + IID == Intrinsic::amdgcn_mov_dpp8 || + IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) { Src1 = N->getOperand(2); - if (IID == Intrinsic::amdgcn_writelane || IsPermLane16) + if (IID == Intrinsic::amdgcn_writelane || + IID == Intrinsic::amdgcn_update_dpp || IsPermLane16) Src2 = N->getOperand(3); } - if (ValSize == 32) { + if (ValSize == SplitSize) { // Already legal return SDValue(); } @@ -6222,7 +6231,7 @@ static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N, Src0 = DAG.getAnyExtOrTrunc(IsFloat ? DAG.getBitcast(IntVT, Src0) : Src0, SL, MVT::i32); - if (IsSetInactive || IsPermLane16) { + if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) { Src1 = DAG.getAnyExtOrTrunc(IsFloat ? DAG.getBitcast(IntVT, Src1) : Src1, SL, MVT::i32); } @@ -6237,7 +6246,7 @@ static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N, return IsFloat ? DAG.getBitcast(VT, Trunc) : Trunc; } - if (ValSize % 32 != 0) + if (ValSize % SplitSize != 0) return SDValue(); auto unrollLaneOp = [&DAG, &SL](SDNode *N) -> SDValue { @@ -6284,21 +6293,26 @@ static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N, switch (MVT::SimpleValueType EltTy = VT.getVectorElementType().getSimpleVT().SimpleTy) { case MVT::i32: - case MVT::f32: { - SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VT.getSimpleVT()); - return unrollLaneOp(LaneOp.getNode()); - } + case MVT::f32: + if (SplitSize == 32) { + SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VT.getSimpleVT()); + return unrollLaneOp(LaneOp.getNode()); + } + [[fallthrough]]; case MVT::i16: case MVT::f16: case MVT::bf16: { - MVT SubVecVT = MVT::getVectorVT(EltTy, 2); + unsigned SubVecNumElt = + SplitSize / VT.getVectorElementType().getSizeInBits(); + MVT SubVecVT = MVT::getVectorVT(EltTy, SubVecNumElt); SmallVector Pieces; SDValue Src0SubVec, Src1SubVec, Src2SubVec; - for (unsigned i = 0, EltIdx = 0; i < ValSize / 32; i++) { + for (unsigned i = 0, EltIdx = 0; i < ValSize / SplitSize; i++) { Src0SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src0, DAG.getConstant(EltIdx, SL, MVT::i32)); - if (IsSetInactive || IsPermLane16) + if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || + IsPermLane16) Src1SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src1, DAG.getConstant(EltIdx, SL, MVT::i32)); @@ -6307,10 +6321,10 @@ static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N, DAG.getConstant(EltIdx, SL, MVT::i32)); Pieces.push_back( - IsSetInactive || IsPermLane16 + IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16 ? createLaneOp(Src0SubVec, Src1SubVec, Src2, SubVecVT) : createLaneOp(Src0SubVec, Src1, Src2SubVec, SubVecVT)); - EltIdx += 2; + EltIdx += SubVecNumElt; } return DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, Pieces); } @@ -6320,10 +6334,11 @@ static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N, } } - MVT VecVT = MVT::getVectorVT(MVT::i32, ValSize / 32); + MVT VecVT = + MVT::getVectorVT(MVT::getIntegerVT(SplitSize), ValSize / SplitSize); Src0 = DAG.getBitcast(VecVT, Src0); - if (IsSetInactive || IsPermLane16) + if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) Src1 = DAG.getBitcast(VecVT, Src1); if (IID == Intrinsic::amdgcn_writelane) @@ -8833,6 +8848,7 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, case Intrinsic::amdgcn_set_inactive: case Intrinsic::amdgcn_set_inactive_chain_arg: case Intrinsic::amdgcn_mov_dpp8: + case Intrinsic::amdgcn_update_dpp: return lowerLaneOp(*this, Op.getNode(), DAG); default: if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr = diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll index 4b0f2ef77a9834..336a7767ee0622 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll @@ -52,11 +52,11 @@ define amdgpu_kernel void @update_dppi64_test(ptr addrspace(1) %arg, i64 %in1, i ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] -; GFX8-NEXT: v_mov_b32_e32 v5, s3 ; GFX8-NEXT: v_mov_b32_e32 v4, s2 +; GFX8-NEXT: v_mov_b32_e32 v5, s3 ; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_mov_b32_dpp v5, v3 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 ; GFX8-NEXT: v_mov_b32_dpp v4, v2 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 +; GFX8-NEXT: v_mov_b32_dpp v5, v3 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 ; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[4:5] ; GFX8-NEXT: s_endpgm ; @@ -77,10 +77,10 @@ define amdgpu_kernel void @update_dppi64_test(ptr addrspace(1) %arg, i64 %in1, i ; GFX11-LABEL: update_dppi64_test: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x24 -; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 +; GFX11-NEXT: v_dual_mov_b32 v3, s3 :: v_dual_and_b32 v0, 0x3ff, v0 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v0 +; GFX11-NEXT: v_mov_b32_e32 v2, s2 ; GFX11-NEXT: global_load_b64 v[0:1], v4, s[0:1] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 @@ -106,11 +106,11 @@ define amdgpu_kernel void @update_dppf64_test(ptr addrspace(1) %arg, double %in1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] -; GFX8-NEXT: v_mov_b32_e32 v5, s3 ; GFX8-NEXT: v_mov_b32_e32 v4, s2 +; GFX8-NEXT: v_mov_b32_e32 v5, s3 ; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_mov_b32_dpp v5, v3 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 ; GFX8-NEXT: v_mov_b32_dpp v4, v2 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 +; GFX8-NEXT: v_mov_b32_dpp v5, v3 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 ; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[4:5] ; GFX8-NEXT: s_endpgm ; @@ -131,10 +131,10 @@ define amdgpu_kernel void @update_dppf64_test(ptr addrspace(1) %arg, double %in1 ; GFX11-LABEL: update_dppf64_test: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x24 -; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 +; GFX11-NEXT: v_dual_mov_b32 v3, s3 :: v_dual_and_b32 v0, 0x3ff, v0 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v0 +; GFX11-NEXT: v_mov_b32_e32 v2, s2 ; GFX11-NEXT: global_load_b64 v[0:1], v4, s[0:1] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 @@ -160,11 +160,11 @@ define amdgpu_kernel void @update_dppv2i32_test(ptr addrspace(1) %arg, <2 x i32> ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] -; GFX8-NEXT: v_mov_b32_e32 v5, s3 ; GFX8-NEXT: v_mov_b32_e32 v4, s2 +; GFX8-NEXT: v_mov_b32_e32 v5, s3 ; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_mov_b32_dpp v5, v3 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 ; GFX8-NEXT: v_mov_b32_dpp v4, v2 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 +; GFX8-NEXT: v_mov_b32_dpp v5, v3 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 ; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[4:5] ; GFX8-NEXT: s_endpgm ; @@ -185,10 +185,10 @@ define amdgpu_kernel void @update_dppv2i32_test(ptr addrspace(1) %arg, <2 x i32> ; GFX11-LABEL: update_dppv2i32_test: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x24 -; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 +; GFX11-NEXT: v_dual_mov_b32 v3, s3 :: v_dual_and_b32 v0, 0x3ff, v0 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v0 +; GFX11-NEXT: v_mov_b32_e32 v2, s2 ; GFX11-NEXT: global_load_b64 v[0:1], v4, s[0:1] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 @@ -214,11 +214,11 @@ define amdgpu_kernel void @update_dppv2f32_test(ptr addrspace(1) %arg, <2 x floa ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] -; GFX8-NEXT: v_mov_b32_e32 v5, s3 ; GFX8-NEXT: v_mov_b32_e32 v4, s2 +; GFX8-NEXT: v_mov_b32_e32 v5, s3 ; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_mov_b32_dpp v5, v3 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 ; GFX8-NEXT: v_mov_b32_dpp v4, v2 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 +; GFX8-NEXT: v_mov_b32_dpp v5, v3 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 ; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[4:5] ; GFX8-NEXT: s_endpgm ; @@ -239,10 +239,10 @@ define amdgpu_kernel void @update_dppv2f32_test(ptr addrspace(1) %arg, <2 x floa ; GFX11-LABEL: update_dppv2f32_test: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x24 -; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 +; GFX11-NEXT: v_dual_mov_b32 v3, s3 :: v_dual_and_b32 v0, 0x3ff, v0 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v0 +; GFX11-NEXT: v_mov_b32_e32 v2, s2 ; GFX11-NEXT: global_load_b64 v[0:1], v4, s[0:1] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 @@ -268,11 +268,11 @@ define amdgpu_kernel void @update_dpp_p0_test(ptr addrspace(1) %arg, ptr %in1, p ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] -; GFX8-NEXT: v_mov_b32_e32 v5, s3 ; GFX8-NEXT: v_mov_b32_e32 v4, s2 +; GFX8-NEXT: v_mov_b32_e32 v5, s3 ; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_mov_b32_dpp v5, v3 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 ; GFX8-NEXT: v_mov_b32_dpp v4, v2 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 +; GFX8-NEXT: v_mov_b32_dpp v5, v3 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 ; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[4:5] ; GFX8-NEXT: s_endpgm ; @@ -293,10 +293,10 @@ define amdgpu_kernel void @update_dpp_p0_test(ptr addrspace(1) %arg, ptr %in1, p ; GFX11-LABEL: update_dpp_p0_test: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x24 -; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 +; GFX11-NEXT: v_dual_mov_b32 v3, s3 :: v_dual_and_b32 v0, 0x3ff, v0 ; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v0 +; GFX11-NEXT: v_mov_b32_e32 v2, s2 ; GFX11-NEXT: global_load_b64 v[0:1], v4, s[0:1] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll index 495bfec5454ee8..98d2e71f3975c2 100644 --- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll +++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll @@ -2819,343 +2819,327 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX8_DPP-LABEL: add_i64_varying: ; GFX8_DPP: ; %bb.0: ; %entry ; GFX8_DPP-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24 -; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 -; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v7 ; GFX8_DPP-NEXT: s_or_saveexec_b64 s[4:5], -1 +; GFX8_DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX8_DPP-NEXT: s_mov_b64 exec, s[4:5] +; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0 +; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v6 +; GFX8_DPP-NEXT: s_or_saveexec_b64 s[4:5], -1 +; GFX8_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[4:5] +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[4:5] +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v5 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v5 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v5 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v5 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v5 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v5 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_addc_u32_e32 v4, vcc, v2, v4, vcc ; GFX8_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 -; GFX8_DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[4:5] -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[4:5] -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: v_readlane_b32 s7, v1, 63 -; GFX8_DPP-NEXT: v_readlane_b32 s6, v4, 63 +; GFX8_DPP-NEXT: v_readlane_b32 s7, v4, 63 +; GFX8_DPP-NEXT: v_readlane_b32 s6, v3, 63 ; GFX8_DPP-NEXT: v_mov_b32_dpp v2, v4 wave_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v1, v3 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX8_DPP-NEXT: s_mov_b64 exec, s[4:5] -; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 -; GFX8_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 +; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6 +; GFX8_DPP-NEXT: ; implicit-def: $vgpr6_vgpr7 ; GFX8_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GFX8_DPP-NEXT: s_cbranch_execz .LBB5_2 ; GFX8_DPP-NEXT: ; %bb.1: -; GFX8_DPP-NEXT: v_mov_b32_e32 v8, s7 +; GFX8_DPP-NEXT: v_mov_b32_e32 v6, s6 ; GFX8_DPP-NEXT: s_mov_b32 s11, 0xf000 ; GFX8_DPP-NEXT: s_mov_b32 s10, -1 ; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX8_DPP-NEXT: s_mov_b32 s8, s2 ; GFX8_DPP-NEXT: s_mov_b32 s9, s3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v7, s6 -; GFX8_DPP-NEXT: buffer_atomic_add_x2 v[7:8], off, s[8:11], 0 glc +; GFX8_DPP-NEXT: v_mov_b32_e32 v7, s7 +; GFX8_DPP-NEXT: buffer_atomic_add_x2 v[6:7], off, s[8:11], 0 glc ; GFX8_DPP-NEXT: s_waitcnt vmcnt(0) ; GFX8_DPP-NEXT: buffer_wbinvl1_vol ; GFX8_DPP-NEXT: .LBB5_2: ; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v8 -; GFX8_DPP-NEXT: v_readfirstlane_b32 s5, v7 +; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v7 +; GFX8_DPP-NEXT: v_readfirstlane_b32 s5, v6 +; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v1 ; GFX8_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX8_DPP-NEXT: v_mov_b32_e32 v8, v3 ; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s4 -; GFX8_DPP-NEXT: v_add_u32_e32 v7, vcc, s5, v7 +; GFX8_DPP-NEXT: v_add_u32_e32 v6, vcc, s5, v6 ; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX8_DPP-NEXT: s_mov_b32 s3, 0xf000 ; GFX8_DPP-NEXT: s_mov_b32 s2, -1 -; GFX8_DPP-NEXT: v_addc_u32_e32 v8, vcc, v0, v8, vcc -; GFX8_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[0:3], 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v7, vcc, v0, v7, vcc +; GFX8_DPP-NEXT: buffer_store_dwordx2 v[6:7], off, s[0:3], 0 ; GFX8_DPP-NEXT: s_endpgm ; ; GFX9_DPP-LABEL: add_i64_varying: ; GFX9_DPP: ; %bb.0: ; %entry ; GFX9_DPP-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24 -; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 -; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v7 ; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 +; GFX9_DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1] +; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0 +; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v6 +; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 +; GFX9_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[0:1] +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v5 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v5 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v5 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v5 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v5 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v5 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v4, vcc, v2, v4, vcc ; GFX9_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 -; GFX9_DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1] -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: v_readlane_b32 s3, v1, 63 -; GFX9_DPP-NEXT: v_readlane_b32 s2, v4, 63 +; GFX9_DPP-NEXT: v_readlane_b32 s3, v4, 63 +; GFX9_DPP-NEXT: v_readlane_b32 s2, v3, 63 ; GFX9_DPP-NEXT: v_mov_b32_dpp v2, v4 wave_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v1, v3 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 -; GFX9_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 +; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6 +; GFX9_DPP-NEXT: ; implicit-def: $vgpr6_vgpr7 ; GFX9_DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX9_DPP-NEXT: s_cbranch_execz .LBB5_2 ; GFX9_DPP-NEXT: ; %bb.1: -; GFX9_DPP-NEXT: v_mov_b32_e32 v8, s3 +; GFX9_DPP-NEXT: v_mov_b32_e32 v7, s3 ; GFX9_DPP-NEXT: s_mov_b32 s11, 0xf000 ; GFX9_DPP-NEXT: s_mov_b32 s10, -1 ; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9_DPP-NEXT: s_mov_b32 s8, s6 ; GFX9_DPP-NEXT: s_mov_b32 s9, s7 -; GFX9_DPP-NEXT: v_mov_b32_e32 v7, s2 -; GFX9_DPP-NEXT: buffer_atomic_add_x2 v[7:8], off, s[8:11], 0 glc +; GFX9_DPP-NEXT: v_mov_b32_e32 v6, s2 +; GFX9_DPP-NEXT: buffer_atomic_add_x2 v[6:7], off, s[8:11], 0 glc ; GFX9_DPP-NEXT: s_waitcnt vmcnt(0) ; GFX9_DPP-NEXT: buffer_wbinvl1_vol ; GFX9_DPP-NEXT: .LBB5_2: ; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[0:1] -; GFX9_DPP-NEXT: v_readfirstlane_b32 s0, v8 -; GFX9_DPP-NEXT: v_readfirstlane_b32 s1, v7 +; GFX9_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX9_DPP-NEXT: v_readfirstlane_b32 s1, v6 +; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v1 ; GFX9_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX9_DPP-NEXT: v_mov_b32_e32 v8, v3 ; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s0 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v7, vcc, s1, v7 +; GFX9_DPP-NEXT: v_add_co_u32_e32 v6, vcc, s1, v6 ; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9_DPP-NEXT: s_mov_b32 s7, 0xf000 ; GFX9_DPP-NEXT: s_mov_b32 s6, -1 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v8, vcc, v0, v8, vcc -; GFX9_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[4:7], 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v7, vcc, v0, v7, vcc +; GFX9_DPP-NEXT: buffer_store_dwordx2 v[6:7], off, s[4:7], 0 ; GFX9_DPP-NEXT: s_endpgm ; ; GFX1064_DPP-LABEL: add_i64_varying: ; GFX1064_DPP: ; %bb.0: ; %entry ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1064_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] -; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, v1 -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v5, 0, 0, s[0:1] -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v2 +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s[0:1] +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v5 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v4 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v5, v6, vcc -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v7 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v8, vcc -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v5 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v6, vcc -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v7 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v8, vcc -; GFX1064_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1064_DPP-NEXT: v_permlanex16_b32 v8, v4, -1, -1 +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v2, v1 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v3, v4, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v6 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v4 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v3, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v6 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1064_DPP-NEXT: v_permlanex16_b32 v6, v2, -1, -1 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v5 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v7, vcc -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v3, 31 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v4, 31 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, s4 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s5 +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v1, 31 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v2, 31 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, s4 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, s5 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf ; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v8 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v5 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v7, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064_DPP-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24 ; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_readlane_b32 s2, v4, 15 -; GFX1064_DPP-NEXT: v_readlane_b32 s3, v3, 15 -; GFX1064_DPP-NEXT: v_readlane_b32 s8, v4, 31 -; GFX1064_DPP-NEXT: v_readlane_b32 s9, v3, 31 -; GFX1064_DPP-NEXT: v_readlane_b32 s10, v3, 47 -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s2, 16 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s3, 16 -; GFX1064_DPP-NEXT: v_readlane_b32 s2, v3, 63 -; GFX1064_DPP-NEXT: v_readlane_b32 s11, v4, 47 -; GFX1064_DPP-NEXT: v_readlane_b32 s3, v4, 63 -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s8, 32 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s9, 32 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_readlane_b32 s2, v2, 15 +; GFX1064_DPP-NEXT: v_readlane_b32 s3, v1, 15 +; GFX1064_DPP-NEXT: v_readlane_b32 s8, v2, 31 +; GFX1064_DPP-NEXT: v_readlane_b32 s9, v1, 31 +; GFX1064_DPP-NEXT: v_readlane_b32 s10, v1, 47 +; GFX1064_DPP-NEXT: v_writelane_b32 v7, s2, 16 +; GFX1064_DPP-NEXT: v_writelane_b32 v6, s3, 16 +; GFX1064_DPP-NEXT: v_readlane_b32 s2, v1, 63 +; GFX1064_DPP-NEXT: v_readlane_b32 s11, v2, 47 +; GFX1064_DPP-NEXT: v_readlane_b32 s3, v2, 63 +; GFX1064_DPP-NEXT: v_writelane_b32 v7, s8, 32 +; GFX1064_DPP-NEXT: v_writelane_b32 v6, s9, 32 ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[8:9], -1 ; GFX1064_DPP-NEXT: s_mov_b64 s[0:1], s[2:3] -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s11, 48 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s10, 48 +; GFX1064_DPP-NEXT: v_writelane_b32 v7, s11, 48 +; GFX1064_DPP-NEXT: v_writelane_b32 v6, s10, 48 ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[8:9] ; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 ; GFX1064_DPP-NEXT: s_mov_b32 s2, -1 -; GFX1064_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10 +; GFX1064_DPP-NEXT: ; implicit-def: $vgpr8_vgpr9 ; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc ; GFX1064_DPP-NEXT: s_cbranch_execz .LBB5_2 ; GFX1064_DPP-NEXT: ; %bb.1: -; GFX1064_DPP-NEXT: v_mov_b32_e32 v10, s1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, s0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, s1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s0 ; GFX1064_DPP-NEXT: s_mov_b32 s3, 0x31016000 ; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064_DPP-NEXT: s_mov_b32 s0, s6 ; GFX1064_DPP-NEXT: s_mov_b32 s1, s7 -; GFX1064_DPP-NEXT: buffer_atomic_add_x2 v[9:10], off, s[0:3], 0 glc +; GFX1064_DPP-NEXT: buffer_atomic_add_x2 v[8:9], off, s[0:3], 0 glc ; GFX1064_DPP-NEXT: s_waitcnt vmcnt(0) ; GFX1064_DPP-NEXT: buffer_gl1_inv ; GFX1064_DPP-NEXT: buffer_gl0_inv ; GFX1064_DPP-NEXT: .LBB5_2: ; GFX1064_DPP-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[8:9] -; GFX1064_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v11, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v12, v2 -; GFX1064_DPP-NEXT: v_readfirstlane_b32 s1, v10 +; GFX1064_DPP-NEXT: v_readfirstlane_b32 s0, v8 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v10, v6 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v11, v7 +; GFX1064_DPP-NEXT: v_readfirstlane_b32 s1, v9 ; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064_DPP-NEXT: s_mov_b32 s7, 0x31016000 -; GFX1064_DPP-NEXT: v_add_co_u32 v9, vcc, s0, v11 +; GFX1064_DPP-NEXT: v_add_co_u32 v8, vcc, s0, v10 ; GFX1064_DPP-NEXT: s_mov_b32 s6, s2 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v10, vcc, s1, v12, vcc -; GFX1064_DPP-NEXT: buffer_store_dwordx2 v[9:10], off, s[4:7], 0 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v9, vcc, s1, v11, vcc +; GFX1064_DPP-NEXT: buffer_store_dwordx2 v[8:9], off, s[4:7], 0 ; GFX1064_DPP-NEXT: s_endpgm ; ; GFX1032_DPP-LABEL: add_i64_varying: ; GFX1032_DPP: ; %bb.0: ; %entry ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1 ; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s0 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, v1 -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v5, 0, 0, s0 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2 +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0 +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v5 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v4 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v5, v6, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v7 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v8, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v5 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v6, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v7 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v8, vcc_lo -; GFX1032_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1032_DPP-NEXT: v_permlanex16_b32 v8, v4, -1, -1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v2, v1 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v3, v4, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v6 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v4 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v3, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v6 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1032_DPP-NEXT: v_permlanex16_b32 v6, v2, -1, -1 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1032_DPP-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24 ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1 -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v5 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v7, vcc_lo -; GFX1032_DPP-NEXT: v_readlane_b32 s0, v3, 31 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_readlane_b32 s8, v4, 15 -; GFX1032_DPP-NEXT: v_readlane_b32 s1, v4, 31 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_readlane_b32 s3, v3, 15 +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo +; GFX1032_DPP-NEXT: v_readlane_b32 s0, v1, 31 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_readlane_b32 s8, v2, 15 +; GFX1032_DPP-NEXT: v_readlane_b32 s1, v2, 31 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_readlane_b32 s3, v1, 15 ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2 ; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1 -; GFX1032_DPP-NEXT: v_writelane_b32 v2, s8, 16 -; GFX1032_DPP-NEXT: v_writelane_b32 v1, s3, 16 +; GFX1032_DPP-NEXT: v_writelane_b32 v8, s8, 16 +; GFX1032_DPP-NEXT: v_writelane_b32 v7, s3, 16 ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2 ; GFX1032_DPP-NEXT: s_mov_b32 s2, -1 ; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 @@ -3177,8 +3161,8 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX1032_DPP-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s8 ; GFX1032_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v11, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v12, v2 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v11, v7 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v12, v8 ; GFX1032_DPP-NEXT: v_readfirstlane_b32 s1, v10 ; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032_DPP-NEXT: s_mov_b32 s7, 0x31016000 @@ -3190,171 +3174,164 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace( ; ; GFX1164_DPP-LABEL: add_i64_varying: ; GFX1164_DPP: ; %bb.0: ; %entry -; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, v2 -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc, v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v4, vcc -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc, v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v2, vcc +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v6, vcc +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc, v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v4, vcc -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v2, vcc +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v6, vcc -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v7, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v4, 31 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s5 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v5, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v7, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v3, 31 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, s4 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v5, vcc +; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v1, 31 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s4 +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v2, 31 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v2, s4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: s_load_b128 s[0:3], s[2:3], 0x24 ; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[4:5], -1 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v1, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_readlane_b32 s6, v4, 15 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_readlane_b32 s7, v3, 15 -; GFX1164_DPP-NEXT: v_readlane_b32 s8, v4, 31 -; GFX1164_DPP-NEXT: v_readlane_b32 s9, v3, 31 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s6, 16 -; GFX1164_DPP-NEXT: v_readlane_b32 s6, v4, 63 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s7, 16 -; GFX1164_DPP-NEXT: v_readlane_b32 s10, v4, 47 -; GFX1164_DPP-NEXT: v_readlane_b32 s11, v3, 47 -; GFX1164_DPP-NEXT: v_readlane_b32 s7, v3, 63 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s8, 32 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s9, 32 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_readlane_b32 s6, v2, 15 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 15 +; GFX1164_DPP-NEXT: v_readlane_b32 s8, v2, 31 +; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 31 +; GFX1164_DPP-NEXT: v_writelane_b32 v4, s6, 16 +; GFX1164_DPP-NEXT: v_readlane_b32 s6, v2, 63 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s7, 16 +; GFX1164_DPP-NEXT: v_readlane_b32 s10, v2, 47 +; GFX1164_DPP-NEXT: v_readlane_b32 s11, v1, 47 +; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 63 +; GFX1164_DPP-NEXT: v_writelane_b32 v4, s8, 32 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s9, 32 ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[4:5] ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[8:9], -1 ; GFX1164_DPP-NEXT: s_mov_b64 s[4:5], s[6:7] -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s10, 48 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s11, 48 +; GFX1164_DPP-NEXT: v_writelane_b32 v4, s10, 48 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s11, 48 ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[8:9] ; GFX1164_DPP-NEXT: s_mov_b32 s6, -1 ; GFX1164_DPP-NEXT: s_mov_b64 s[8:9], exec -; GFX1164_DPP-NEXT: ; implicit-def: $vgpr8_vgpr9 +; GFX1164_DPP-NEXT: ; implicit-def: $vgpr6_vgpr7 ; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 ; GFX1164_DPP-NEXT: s_cbranch_execz .LBB5_2 ; GFX1164_DPP-NEXT: ; %bb.1: -; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, s5 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, s4 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s5 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, s4 ; GFX1164_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164_DPP-NEXT: s_mov_b32 s4, s2 ; GFX1164_DPP-NEXT: s_mov_b32 s5, s3 -; GFX1164_DPP-NEXT: buffer_atomic_add_u64 v[8:9], off, s[4:7], 0 glc +; GFX1164_DPP-NEXT: buffer_atomic_add_u64 v[6:7], off, s[4:7], 0 glc ; GFX1164_DPP-NEXT: s_waitcnt vmcnt(0) ; GFX1164_DPP-NEXT: buffer_gl1_inv ; GFX1164_DPP-NEXT: buffer_gl0_inv ; GFX1164_DPP-NEXT: .LBB5_2: ; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[8:9] ; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1164_DPP-NEXT: v_readfirstlane_b32 s2, v8 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v11, v2 -; GFX1164_DPP-NEXT: v_readfirstlane_b32 s3, v9 +; GFX1164_DPP-NEXT: v_readfirstlane_b32 s2, v6 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v4 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, v5 +; GFX1164_DPP-NEXT: v_readfirstlane_b32 s3, v7 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_add_co_u32 v8, vcc, s2, v10 +; GFX1164_DPP-NEXT: v_add_co_u32 v6, vcc, s2, v8 ; GFX1164_DPP-NEXT: s_mov_b32 s2, s6 -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v9, vcc, s3, v11, vcc +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v7, vcc, s3, v9, vcc ; GFX1164_DPP-NEXT: s_mov_b32 s3, 0x31016000 -; GFX1164_DPP-NEXT: buffer_store_b64 v[8:9], off, s[0:3], 0 +; GFX1164_DPP-NEXT: buffer_store_b64 v[6:7], off, s[0:3], 0 ; GFX1164_DPP-NEXT: s_endpgm ; ; GFX1132_DPP-LABEL: add_i64_varying: ; GFX1132_DPP: ; %bb.0: ; %entry -; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s0 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, v2 -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc_lo, v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s0 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s0 +; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v7, 0 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc_lo, v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v6, vcc_lo +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc_lo, v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc_lo, v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v6, vcc_lo -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc_lo, v7, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1132_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1132_DPP-NEXT: s_load_b128 s[0:3], s[2:3], 0x24 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s6, -1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v5, vcc_lo -; GFX1132_DPP-NEXT: v_readlane_b32 s4, v4, 31 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v1, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_readlane_b32 s7, v4, 15 -; GFX1132_DPP-NEXT: v_readlane_b32 s8, v3, 15 -; GFX1132_DPP-NEXT: v_readlane_b32 s5, v3, 31 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo +; GFX1132_DPP-NEXT: v_readlane_b32 s4, v2, 31 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_readlane_b32 s7, v2, 15 +; GFX1132_DPP-NEXT: v_readlane_b32 s8, v1, 15 +; GFX1132_DPP-NEXT: v_readlane_b32 s5, v1, 31 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s6 ; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s6, -1 -; GFX1132_DPP-NEXT: v_writelane_b32 v1, s7, 16 -; GFX1132_DPP-NEXT: v_writelane_b32 v2, s8, 16 +; GFX1132_DPP-NEXT: v_writelane_b32 v6, s7, 16 +; GFX1132_DPP-NEXT: v_writelane_b32 v7, s8, 16 ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s6 ; GFX1132_DPP-NEXT: s_mov_b32 s6, -1 ; GFX1132_DPP-NEXT: s_mov_b32 s8, exec_lo @@ -3375,8 +3352,8 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s8 ; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132_DPP-NEXT: v_readfirstlane_b32 s2, v8 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v10, v1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v11, v2 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v10, v6 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v11, v7 ; GFX1132_DPP-NEXT: v_readfirstlane_b32 s3, v9 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX1132_DPP-NEXT: v_add_co_u32 v8, vcc_lo, s2, v10 @@ -3388,173 +3365,166 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace( ; ; GFX1264_DPP-LABEL: add_i64_varying: ; GFX1264_DPP: ; %bb.0: ; %entry -; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1264_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1264_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1264_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1264_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1264_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1264_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s[0:1] -; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, v2 -; GFX1264_DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] -; GFX1264_DPP-NEXT: v_mov_b32_e32 v6, v2 +; GFX1264_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] +; GFX1264_DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1264_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] +; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1264_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc, v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1264_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v4, vcc -; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc, v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v2, vcc +; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1264_DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1264_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v6, vcc +; GFX1264_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc ; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc, v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1264_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1264_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1264_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v4, vcc -; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1264_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v2, vcc +; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1264_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1264_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1264_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v6, vcc -; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v7, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1264_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1264_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 ; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1264_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1264_DPP-NEXT: v_readlane_b32 s5, v4, 31 -; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1264_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1264_DPP-NEXT: v_mov_b32_e32 v7, s5 -; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v5, vcc -; GFX1264_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v7, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc +; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1264_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1264_DPP-NEXT: v_readlane_b32 s4, v3, 31 -; GFX1264_DPP-NEXT: v_mov_b32_e32 v6, s4 -; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1264_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v5, vcc +; GFX1264_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1264_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc +; GFX1264_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1264_DPP-NEXT: v_readlane_b32 s4, v1, 31 +; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, s4 +; GFX1264_DPP-NEXT: v_readlane_b32 s4, v2, 31 +; GFX1264_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v2, s4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc ; GFX1264_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1264_DPP-NEXT: s_load_b128 s[0:3], s[2:3], 0x24 ; GFX1264_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[4:5], -1 -; GFX1264_DPP-NEXT: v_mov_b32_dpp v1, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1264_DPP-NEXT: v_readlane_b32 s6, v4, 15 -; GFX1264_DPP-NEXT: v_mov_b32_dpp v2, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1264_DPP-NEXT: v_readlane_b32 s7, v3, 15 -; GFX1264_DPP-NEXT: v_readlane_b32 s8, v4, 31 -; GFX1264_DPP-NEXT: v_readlane_b32 s9, v3, 31 -; GFX1264_DPP-NEXT: v_writelane_b32 v1, s6, 16 -; GFX1264_DPP-NEXT: v_readlane_b32 s6, v4, 63 -; GFX1264_DPP-NEXT: v_writelane_b32 v2, s7, 16 -; GFX1264_DPP-NEXT: v_readlane_b32 s10, v4, 47 -; GFX1264_DPP-NEXT: v_readlane_b32 s11, v3, 47 -; GFX1264_DPP-NEXT: v_readlane_b32 s7, v3, 63 -; GFX1264_DPP-NEXT: v_writelane_b32 v1, s8, 32 -; GFX1264_DPP-NEXT: v_writelane_b32 v2, s9, 32 +; GFX1264_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1264_DPP-NEXT: v_readlane_b32 s6, v2, 15 +; GFX1264_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1264_DPP-NEXT: v_readlane_b32 s7, v1, 15 +; GFX1264_DPP-NEXT: v_readlane_b32 s8, v2, 31 +; GFX1264_DPP-NEXT: v_readlane_b32 s9, v1, 31 +; GFX1264_DPP-NEXT: v_writelane_b32 v4, s6, 16 +; GFX1264_DPP-NEXT: v_readlane_b32 s6, v2, 63 +; GFX1264_DPP-NEXT: v_writelane_b32 v5, s7, 16 +; GFX1264_DPP-NEXT: v_readlane_b32 s10, v2, 47 +; GFX1264_DPP-NEXT: v_readlane_b32 s11, v1, 47 +; GFX1264_DPP-NEXT: v_readlane_b32 s7, v1, 63 +; GFX1264_DPP-NEXT: v_writelane_b32 v4, s8, 32 +; GFX1264_DPP-NEXT: v_writelane_b32 v5, s9, 32 ; GFX1264_DPP-NEXT: s_wait_alu 0xfffe ; GFX1264_DPP-NEXT: s_mov_b64 exec, s[4:5] ; GFX1264_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1264_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 ; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[8:9], -1 ; GFX1264_DPP-NEXT: s_mov_b64 s[4:5], s[6:7] -; GFX1264_DPP-NEXT: v_writelane_b32 v1, s10, 48 -; GFX1264_DPP-NEXT: v_writelane_b32 v2, s11, 48 +; GFX1264_DPP-NEXT: v_writelane_b32 v4, s10, 48 +; GFX1264_DPP-NEXT: v_writelane_b32 v5, s11, 48 ; GFX1264_DPP-NEXT: s_wait_alu 0xfffe ; GFX1264_DPP-NEXT: s_mov_b64 exec, s[8:9] ; GFX1264_DPP-NEXT: s_mov_b32 s6, -1 ; GFX1264_DPP-NEXT: s_mov_b64 s[8:9], exec -; GFX1264_DPP-NEXT: ; implicit-def: $vgpr8_vgpr9 +; GFX1264_DPP-NEXT: ; implicit-def: $vgpr6_vgpr7 ; GFX1264_DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 ; GFX1264_DPP-NEXT: s_cbranch_execz .LBB5_2 ; GFX1264_DPP-NEXT: ; %bb.1: -; GFX1264_DPP-NEXT: v_mov_b32_e32 v9, s5 -; GFX1264_DPP-NEXT: v_mov_b32_e32 v8, s4 +; GFX1264_DPP-NEXT: v_mov_b32_e32 v7, s5 +; GFX1264_DPP-NEXT: v_mov_b32_e32 v6, s4 ; GFX1264_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1264_DPP-NEXT: s_wait_kmcnt 0x0 ; GFX1264_DPP-NEXT: s_mov_b32 s4, s2 ; GFX1264_DPP-NEXT: s_mov_b32 s5, s3 -; GFX1264_DPP-NEXT: buffer_atomic_add_u64 v[8:9], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1264_DPP-NEXT: buffer_atomic_add_u64 v[6:7], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX1264_DPP-NEXT: s_wait_loadcnt 0x0 ; GFX1264_DPP-NEXT: global_inv scope:SCOPE_DEV ; GFX1264_DPP-NEXT: .LBB5_2: ; GFX1264_DPP-NEXT: s_wait_alu 0xfffe ; GFX1264_DPP-NEXT: s_or_b64 exec, exec, s[8:9] ; GFX1264_DPP-NEXT: s_wait_kmcnt 0x0 -; GFX1264_DPP-NEXT: v_readfirstlane_b32 s2, v8 -; GFX1264_DPP-NEXT: v_mov_b32_e32 v10, v1 -; GFX1264_DPP-NEXT: v_mov_b32_e32 v11, v2 -; GFX1264_DPP-NEXT: v_readfirstlane_b32 s3, v9 +; GFX1264_DPP-NEXT: v_readfirstlane_b32 s2, v6 +; GFX1264_DPP-NEXT: v_mov_b32_e32 v8, v4 +; GFX1264_DPP-NEXT: v_mov_b32_e32 v9, v5 +; GFX1264_DPP-NEXT: v_readfirstlane_b32 s3, v7 ; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX1264_DPP-NEXT: v_add_co_u32 v8, vcc, s2, v10 +; GFX1264_DPP-NEXT: v_add_co_u32 v6, vcc, s2, v8 ; GFX1264_DPP-NEXT: s_mov_b32 s2, s6 -; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v9, vcc, s3, v11, vcc +; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v7, vcc, s3, v9, vcc ; GFX1264_DPP-NEXT: s_mov_b32 s3, 0x31016000 -; GFX1264_DPP-NEXT: buffer_store_b64 v[8:9], off, s[0:3], null +; GFX1264_DPP-NEXT: buffer_store_b64 v[6:7], off, s[0:3], null ; GFX1264_DPP-NEXT: s_endpgm ; ; GFX1232_DPP-LABEL: add_i64_varying: ; GFX1232_DPP: ; %bb.0: ; %entry -; GFX1232_DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1232_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1232_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1232_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1232_DPP-NEXT: s_or_saveexec_b32 s0, -1 ; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1232_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s0 -; GFX1232_DPP-NEXT: v_mov_b32_e32 v4, v2 -; GFX1232_DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0 -; GFX1232_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1232_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc_lo, v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1232_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s0 +; GFX1232_DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1232_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s0 +; GFX1232_DPP-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v7, 0 +; GFX1232_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1232_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo -; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc_lo, v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1232_DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo +; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1232_DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1232_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v6, vcc_lo +; GFX1232_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo ; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc_lo, v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1232_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1232_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1232_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1232_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo -; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc_lo, v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1232_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo +; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1232_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1232_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1232_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 +; GFX1232_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1232_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 ; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v6, vcc_lo -; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc_lo, v7, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo +; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1232_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1232_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1232_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1232_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1232_DPP-NEXT: s_load_b128 s[0:3], s[2:3], 0x24 ; GFX1232_DPP-NEXT: s_or_saveexec_b32 s6, -1 ; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v5, vcc_lo -; GFX1232_DPP-NEXT: v_readlane_b32 s4, v4, 31 -; GFX1232_DPP-NEXT: v_mov_b32_dpp v1, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1232_DPP-NEXT: v_readlane_b32 s7, v4, 15 -; GFX1232_DPP-NEXT: v_readlane_b32 s8, v3, 15 -; GFX1232_DPP-NEXT: v_readlane_b32 s5, v3, 31 -; GFX1232_DPP-NEXT: v_mov_b32_dpp v2, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo +; GFX1232_DPP-NEXT: v_readlane_b32 s4, v2, 31 +; GFX1232_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1232_DPP-NEXT: v_readlane_b32 s7, v2, 15 +; GFX1232_DPP-NEXT: v_readlane_b32 s8, v1, 15 +; GFX1232_DPP-NEXT: v_readlane_b32 s5, v1, 31 +; GFX1232_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s6 ; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1232_DPP-NEXT: s_or_saveexec_b32 s6, -1 -; GFX1232_DPP-NEXT: v_writelane_b32 v1, s7, 16 -; GFX1232_DPP-NEXT: v_writelane_b32 v2, s8, 16 +; GFX1232_DPP-NEXT: v_writelane_b32 v6, s7, 16 +; GFX1232_DPP-NEXT: v_writelane_b32 v7, s8, 16 ; GFX1232_DPP-NEXT: s_wait_alu 0xfffe ; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s6 ; GFX1232_DPP-NEXT: s_mov_b32 s6, -1 @@ -3576,8 +3546,8 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX1232_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s8 ; GFX1232_DPP-NEXT: s_wait_kmcnt 0x0 ; GFX1232_DPP-NEXT: v_readfirstlane_b32 s2, v8 -; GFX1232_DPP-NEXT: v_mov_b32_e32 v10, v1 -; GFX1232_DPP-NEXT: v_mov_b32_e32 v11, v2 +; GFX1232_DPP-NEXT: v_mov_b32_e32 v10, v6 +; GFX1232_DPP-NEXT: v_mov_b32_e32 v11, v7 ; GFX1232_DPP-NEXT: v_readfirstlane_b32 s3, v9 ; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX1232_DPP-NEXT: v_add_co_u32 v8, vcc_lo, s2, v10 @@ -6448,343 +6418,327 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX8_DPP-LABEL: sub_i64_varying: ; GFX8_DPP: ; %bb.0: ; %entry ; GFX8_DPP-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24 -; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 -; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v7 ; GFX8_DPP-NEXT: s_or_saveexec_b64 s[4:5], -1 +; GFX8_DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX8_DPP-NEXT: s_mov_b64 exec, s[4:5] +; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0 +; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v6 +; GFX8_DPP-NEXT: s_or_saveexec_b64 s[4:5], -1 +; GFX8_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[4:5] +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[4:5] +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v5 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v5 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v5 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v5 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v5 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v5 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_addc_u32_e32 v4, vcc, v2, v4, vcc ; GFX8_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 -; GFX8_DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[4:5] -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[4:5] -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: v_readlane_b32 s7, v1, 63 -; GFX8_DPP-NEXT: v_readlane_b32 s6, v4, 63 +; GFX8_DPP-NEXT: v_readlane_b32 s7, v4, 63 +; GFX8_DPP-NEXT: v_readlane_b32 s6, v3, 63 ; GFX8_DPP-NEXT: v_mov_b32_dpp v2, v4 wave_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v1, v3 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX8_DPP-NEXT: s_mov_b64 exec, s[4:5] -; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 -; GFX8_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 +; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6 +; GFX8_DPP-NEXT: ; implicit-def: $vgpr6_vgpr7 ; GFX8_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GFX8_DPP-NEXT: s_cbranch_execz .LBB11_2 ; GFX8_DPP-NEXT: ; %bb.1: -; GFX8_DPP-NEXT: v_mov_b32_e32 v8, s7 +; GFX8_DPP-NEXT: v_mov_b32_e32 v6, s6 ; GFX8_DPP-NEXT: s_mov_b32 s11, 0xf000 ; GFX8_DPP-NEXT: s_mov_b32 s10, -1 ; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX8_DPP-NEXT: s_mov_b32 s8, s2 ; GFX8_DPP-NEXT: s_mov_b32 s9, s3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v7, s6 -; GFX8_DPP-NEXT: buffer_atomic_sub_x2 v[7:8], off, s[8:11], 0 glc +; GFX8_DPP-NEXT: v_mov_b32_e32 v7, s7 +; GFX8_DPP-NEXT: buffer_atomic_sub_x2 v[6:7], off, s[8:11], 0 glc ; GFX8_DPP-NEXT: s_waitcnt vmcnt(0) ; GFX8_DPP-NEXT: buffer_wbinvl1_vol ; GFX8_DPP-NEXT: .LBB11_2: ; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v8 -; GFX8_DPP-NEXT: v_readfirstlane_b32 s5, v7 +; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v7 +; GFX8_DPP-NEXT: v_readfirstlane_b32 s5, v6 +; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v1 ; GFX8_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX8_DPP-NEXT: v_mov_b32_e32 v8, v3 ; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s4 -; GFX8_DPP-NEXT: v_sub_u32_e32 v7, vcc, s5, v7 +; GFX8_DPP-NEXT: v_sub_u32_e32 v6, vcc, s5, v6 ; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX8_DPP-NEXT: s_mov_b32 s3, 0xf000 ; GFX8_DPP-NEXT: s_mov_b32 s2, -1 -; GFX8_DPP-NEXT: v_subb_u32_e32 v8, vcc, v0, v8, vcc -; GFX8_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[0:3], 0 +; GFX8_DPP-NEXT: v_subb_u32_e32 v7, vcc, v0, v7, vcc +; GFX8_DPP-NEXT: buffer_store_dwordx2 v[6:7], off, s[0:3], 0 ; GFX8_DPP-NEXT: s_endpgm ; ; GFX9_DPP-LABEL: sub_i64_varying: ; GFX9_DPP: ; %bb.0: ; %entry ; GFX9_DPP-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24 -; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 -; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v7 ; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 +; GFX9_DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1] +; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0 +; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v6 +; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 +; GFX9_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[0:1] +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v5 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v5 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v5 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v5 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v5 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v5 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v4, vcc, v2, v4, vcc ; GFX9_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 -; GFX9_DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1] -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: v_readlane_b32 s3, v1, 63 -; GFX9_DPP-NEXT: v_readlane_b32 s2, v4, 63 +; GFX9_DPP-NEXT: v_readlane_b32 s3, v4, 63 +; GFX9_DPP-NEXT: v_readlane_b32 s2, v3, 63 ; GFX9_DPP-NEXT: v_mov_b32_dpp v2, v4 wave_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v1, v3 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 -; GFX9_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 +; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6 +; GFX9_DPP-NEXT: ; implicit-def: $vgpr6_vgpr7 ; GFX9_DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX9_DPP-NEXT: s_cbranch_execz .LBB11_2 ; GFX9_DPP-NEXT: ; %bb.1: -; GFX9_DPP-NEXT: v_mov_b32_e32 v8, s3 +; GFX9_DPP-NEXT: v_mov_b32_e32 v7, s3 ; GFX9_DPP-NEXT: s_mov_b32 s11, 0xf000 ; GFX9_DPP-NEXT: s_mov_b32 s10, -1 ; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9_DPP-NEXT: s_mov_b32 s8, s6 ; GFX9_DPP-NEXT: s_mov_b32 s9, s7 -; GFX9_DPP-NEXT: v_mov_b32_e32 v7, s2 -; GFX9_DPP-NEXT: buffer_atomic_sub_x2 v[7:8], off, s[8:11], 0 glc +; GFX9_DPP-NEXT: v_mov_b32_e32 v6, s2 +; GFX9_DPP-NEXT: buffer_atomic_sub_x2 v[6:7], off, s[8:11], 0 glc ; GFX9_DPP-NEXT: s_waitcnt vmcnt(0) ; GFX9_DPP-NEXT: buffer_wbinvl1_vol ; GFX9_DPP-NEXT: .LBB11_2: ; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[0:1] -; GFX9_DPP-NEXT: v_readfirstlane_b32 s0, v8 -; GFX9_DPP-NEXT: v_readfirstlane_b32 s1, v7 +; GFX9_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX9_DPP-NEXT: v_readfirstlane_b32 s1, v6 +; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v1 ; GFX9_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX9_DPP-NEXT: v_mov_b32_e32 v8, v3 ; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s0 -; GFX9_DPP-NEXT: v_sub_co_u32_e32 v7, vcc, s1, v7 +; GFX9_DPP-NEXT: v_sub_co_u32_e32 v6, vcc, s1, v6 ; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9_DPP-NEXT: s_mov_b32 s7, 0xf000 ; GFX9_DPP-NEXT: s_mov_b32 s6, -1 -; GFX9_DPP-NEXT: v_subb_co_u32_e32 v8, vcc, v0, v8, vcc -; GFX9_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[4:7], 0 +; GFX9_DPP-NEXT: v_subb_co_u32_e32 v7, vcc, v0, v7, vcc +; GFX9_DPP-NEXT: buffer_store_dwordx2 v[6:7], off, s[4:7], 0 ; GFX9_DPP-NEXT: s_endpgm ; ; GFX1064_DPP-LABEL: sub_i64_varying: ; GFX1064_DPP: ; %bb.0: ; %entry ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1064_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] -; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, v1 -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v5, 0, 0, s[0:1] -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v2 +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s[0:1] +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v5 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v4 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v5, v6, vcc -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v7 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v8, vcc -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v5 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v6, vcc -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v7 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v8, vcc -; GFX1064_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1064_DPP-NEXT: v_permlanex16_b32 v8, v4, -1, -1 +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v2, v1 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v3, v4, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v6 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v4 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v3, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v6 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1064_DPP-NEXT: v_permlanex16_b32 v6, v2, -1, -1 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v5 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v7, vcc -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v3, 31 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v4, 31 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, s4 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s5 +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v1, 31 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v2, 31 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, s4 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, s5 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf ; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v8 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v5 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v7, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064_DPP-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24 ; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_readlane_b32 s2, v4, 15 -; GFX1064_DPP-NEXT: v_readlane_b32 s3, v3, 15 -; GFX1064_DPP-NEXT: v_readlane_b32 s8, v4, 31 -; GFX1064_DPP-NEXT: v_readlane_b32 s9, v3, 31 -; GFX1064_DPP-NEXT: v_readlane_b32 s10, v3, 47 -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s2, 16 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s3, 16 -; GFX1064_DPP-NEXT: v_readlane_b32 s2, v3, 63 -; GFX1064_DPP-NEXT: v_readlane_b32 s11, v4, 47 -; GFX1064_DPP-NEXT: v_readlane_b32 s3, v4, 63 -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s8, 32 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s9, 32 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_readlane_b32 s2, v2, 15 +; GFX1064_DPP-NEXT: v_readlane_b32 s3, v1, 15 +; GFX1064_DPP-NEXT: v_readlane_b32 s8, v2, 31 +; GFX1064_DPP-NEXT: v_readlane_b32 s9, v1, 31 +; GFX1064_DPP-NEXT: v_readlane_b32 s10, v1, 47 +; GFX1064_DPP-NEXT: v_writelane_b32 v7, s2, 16 +; GFX1064_DPP-NEXT: v_writelane_b32 v6, s3, 16 +; GFX1064_DPP-NEXT: v_readlane_b32 s2, v1, 63 +; GFX1064_DPP-NEXT: v_readlane_b32 s11, v2, 47 +; GFX1064_DPP-NEXT: v_readlane_b32 s3, v2, 63 +; GFX1064_DPP-NEXT: v_writelane_b32 v7, s8, 32 +; GFX1064_DPP-NEXT: v_writelane_b32 v6, s9, 32 ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[8:9], -1 ; GFX1064_DPP-NEXT: s_mov_b64 s[0:1], s[2:3] -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s11, 48 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s10, 48 +; GFX1064_DPP-NEXT: v_writelane_b32 v7, s11, 48 +; GFX1064_DPP-NEXT: v_writelane_b32 v6, s10, 48 ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[8:9] ; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 ; GFX1064_DPP-NEXT: s_mov_b32 s2, -1 -; GFX1064_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10 +; GFX1064_DPP-NEXT: ; implicit-def: $vgpr8_vgpr9 ; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc ; GFX1064_DPP-NEXT: s_cbranch_execz .LBB11_2 ; GFX1064_DPP-NEXT: ; %bb.1: -; GFX1064_DPP-NEXT: v_mov_b32_e32 v10, s1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, s0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, s1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s0 ; GFX1064_DPP-NEXT: s_mov_b32 s3, 0x31016000 ; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064_DPP-NEXT: s_mov_b32 s0, s6 ; GFX1064_DPP-NEXT: s_mov_b32 s1, s7 -; GFX1064_DPP-NEXT: buffer_atomic_sub_x2 v[9:10], off, s[0:3], 0 glc +; GFX1064_DPP-NEXT: buffer_atomic_sub_x2 v[8:9], off, s[0:3], 0 glc ; GFX1064_DPP-NEXT: s_waitcnt vmcnt(0) ; GFX1064_DPP-NEXT: buffer_gl1_inv ; GFX1064_DPP-NEXT: buffer_gl0_inv ; GFX1064_DPP-NEXT: .LBB11_2: ; GFX1064_DPP-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[8:9] -; GFX1064_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v11, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v12, v2 -; GFX1064_DPP-NEXT: v_readfirstlane_b32 s1, v10 +; GFX1064_DPP-NEXT: v_readfirstlane_b32 s0, v8 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v10, v6 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v11, v7 +; GFX1064_DPP-NEXT: v_readfirstlane_b32 s1, v9 ; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064_DPP-NEXT: s_mov_b32 s7, 0x31016000 -; GFX1064_DPP-NEXT: v_sub_co_u32 v9, vcc, s0, v11 +; GFX1064_DPP-NEXT: v_sub_co_u32 v8, vcc, s0, v10 ; GFX1064_DPP-NEXT: s_mov_b32 s6, s2 -; GFX1064_DPP-NEXT: v_sub_co_ci_u32_e32 v10, vcc, s1, v12, vcc -; GFX1064_DPP-NEXT: buffer_store_dwordx2 v[9:10], off, s[4:7], 0 +; GFX1064_DPP-NEXT: v_sub_co_ci_u32_e32 v9, vcc, s1, v11, vcc +; GFX1064_DPP-NEXT: buffer_store_dwordx2 v[8:9], off, s[4:7], 0 ; GFX1064_DPP-NEXT: s_endpgm ; ; GFX1032_DPP-LABEL: sub_i64_varying: ; GFX1032_DPP: ; %bb.0: ; %entry ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1 ; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s0 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, v1 -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v5, 0, 0, s0 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2 +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0 +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v5 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v4 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v5, v6, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v7 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v8, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v5 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v6, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v7 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v8, vcc_lo -; GFX1032_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1032_DPP-NEXT: v_permlanex16_b32 v8, v4, -1, -1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v2, v1 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v3, v4, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v6 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v4 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v3, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v6 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1032_DPP-NEXT: v_permlanex16_b32 v6, v2, -1, -1 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1032_DPP-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24 ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1 -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v5 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v7, vcc_lo -; GFX1032_DPP-NEXT: v_readlane_b32 s0, v3, 31 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_readlane_b32 s8, v4, 15 -; GFX1032_DPP-NEXT: v_readlane_b32 s1, v4, 31 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_readlane_b32 s3, v3, 15 +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo +; GFX1032_DPP-NEXT: v_readlane_b32 s0, v1, 31 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_readlane_b32 s8, v2, 15 +; GFX1032_DPP-NEXT: v_readlane_b32 s1, v2, 31 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_readlane_b32 s3, v1, 15 ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2 ; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1 -; GFX1032_DPP-NEXT: v_writelane_b32 v2, s8, 16 -; GFX1032_DPP-NEXT: v_writelane_b32 v1, s3, 16 +; GFX1032_DPP-NEXT: v_writelane_b32 v8, s8, 16 +; GFX1032_DPP-NEXT: v_writelane_b32 v7, s3, 16 ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2 ; GFX1032_DPP-NEXT: s_mov_b32 s2, -1 ; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 @@ -6806,8 +6760,8 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX1032_DPP-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s8 ; GFX1032_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v11, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v12, v2 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v11, v7 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v12, v8 ; GFX1032_DPP-NEXT: v_readfirstlane_b32 s1, v10 ; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032_DPP-NEXT: s_mov_b32 s7, 0x31016000 @@ -6819,171 +6773,164 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace( ; ; GFX1164_DPP-LABEL: sub_i64_varying: ; GFX1164_DPP: ; %bb.0: ; %entry -; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, v2 -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc, v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v4, vcc -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc, v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v2, vcc +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v6, vcc +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc, v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v4, vcc -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v2, vcc +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v6, vcc -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v7, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v4, 31 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s5 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v5, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v7, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v3, 31 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, s4 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v5, vcc +; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v1, 31 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s4 +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v2, 31 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v2, s4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: s_load_b128 s[0:3], s[2:3], 0x24 ; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[4:5], -1 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v1, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_readlane_b32 s6, v4, 15 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_readlane_b32 s7, v3, 15 -; GFX1164_DPP-NEXT: v_readlane_b32 s8, v4, 31 -; GFX1164_DPP-NEXT: v_readlane_b32 s9, v3, 31 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s6, 16 -; GFX1164_DPP-NEXT: v_readlane_b32 s6, v4, 63 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s7, 16 -; GFX1164_DPP-NEXT: v_readlane_b32 s10, v4, 47 -; GFX1164_DPP-NEXT: v_readlane_b32 s11, v3, 47 -; GFX1164_DPP-NEXT: v_readlane_b32 s7, v3, 63 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s8, 32 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s9, 32 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_readlane_b32 s6, v2, 15 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 15 +; GFX1164_DPP-NEXT: v_readlane_b32 s8, v2, 31 +; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 31 +; GFX1164_DPP-NEXT: v_writelane_b32 v4, s6, 16 +; GFX1164_DPP-NEXT: v_readlane_b32 s6, v2, 63 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s7, 16 +; GFX1164_DPP-NEXT: v_readlane_b32 s10, v2, 47 +; GFX1164_DPP-NEXT: v_readlane_b32 s11, v1, 47 +; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 63 +; GFX1164_DPP-NEXT: v_writelane_b32 v4, s8, 32 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s9, 32 ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[4:5] ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[8:9], -1 ; GFX1164_DPP-NEXT: s_mov_b64 s[4:5], s[6:7] -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s10, 48 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s11, 48 +; GFX1164_DPP-NEXT: v_writelane_b32 v4, s10, 48 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s11, 48 ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[8:9] ; GFX1164_DPP-NEXT: s_mov_b32 s6, -1 ; GFX1164_DPP-NEXT: s_mov_b64 s[8:9], exec -; GFX1164_DPP-NEXT: ; implicit-def: $vgpr8_vgpr9 +; GFX1164_DPP-NEXT: ; implicit-def: $vgpr6_vgpr7 ; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 ; GFX1164_DPP-NEXT: s_cbranch_execz .LBB11_2 ; GFX1164_DPP-NEXT: ; %bb.1: -; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, s5 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, s4 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s5 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, s4 ; GFX1164_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164_DPP-NEXT: s_mov_b32 s4, s2 ; GFX1164_DPP-NEXT: s_mov_b32 s5, s3 -; GFX1164_DPP-NEXT: buffer_atomic_sub_u64 v[8:9], off, s[4:7], 0 glc +; GFX1164_DPP-NEXT: buffer_atomic_sub_u64 v[6:7], off, s[4:7], 0 glc ; GFX1164_DPP-NEXT: s_waitcnt vmcnt(0) ; GFX1164_DPP-NEXT: buffer_gl1_inv ; GFX1164_DPP-NEXT: buffer_gl0_inv ; GFX1164_DPP-NEXT: .LBB11_2: ; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[8:9] ; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1164_DPP-NEXT: v_readfirstlane_b32 s2, v8 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v11, v2 -; GFX1164_DPP-NEXT: v_readfirstlane_b32 s3, v9 +; GFX1164_DPP-NEXT: v_readfirstlane_b32 s2, v6 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v4 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, v5 +; GFX1164_DPP-NEXT: v_readfirstlane_b32 s3, v7 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_sub_co_u32 v8, vcc, s2, v10 +; GFX1164_DPP-NEXT: v_sub_co_u32 v6, vcc, s2, v8 ; GFX1164_DPP-NEXT: s_mov_b32 s2, s6 -; GFX1164_DPP-NEXT: v_sub_co_ci_u32_e32 v9, vcc, s3, v11, vcc +; GFX1164_DPP-NEXT: v_sub_co_ci_u32_e32 v7, vcc, s3, v9, vcc ; GFX1164_DPP-NEXT: s_mov_b32 s3, 0x31016000 -; GFX1164_DPP-NEXT: buffer_store_b64 v[8:9], off, s[0:3], 0 +; GFX1164_DPP-NEXT: buffer_store_b64 v[6:7], off, s[0:3], 0 ; GFX1164_DPP-NEXT: s_endpgm ; ; GFX1132_DPP-LABEL: sub_i64_varying: ; GFX1132_DPP: ; %bb.0: ; %entry -; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s0 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, v2 -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc_lo, v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s0 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s0 +; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v7, 0 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc_lo, v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v6, vcc_lo +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc_lo, v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc_lo, v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v6, vcc_lo -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc_lo, v7, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1132_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1132_DPP-NEXT: s_load_b128 s[0:3], s[2:3], 0x24 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s6, -1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v5, vcc_lo -; GFX1132_DPP-NEXT: v_readlane_b32 s4, v4, 31 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v1, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_readlane_b32 s7, v4, 15 -; GFX1132_DPP-NEXT: v_readlane_b32 s8, v3, 15 -; GFX1132_DPP-NEXT: v_readlane_b32 s5, v3, 31 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo +; GFX1132_DPP-NEXT: v_readlane_b32 s4, v2, 31 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_readlane_b32 s7, v2, 15 +; GFX1132_DPP-NEXT: v_readlane_b32 s8, v1, 15 +; GFX1132_DPP-NEXT: v_readlane_b32 s5, v1, 31 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s6 ; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s6, -1 -; GFX1132_DPP-NEXT: v_writelane_b32 v1, s7, 16 -; GFX1132_DPP-NEXT: v_writelane_b32 v2, s8, 16 +; GFX1132_DPP-NEXT: v_writelane_b32 v6, s7, 16 +; GFX1132_DPP-NEXT: v_writelane_b32 v7, s8, 16 ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s6 ; GFX1132_DPP-NEXT: s_mov_b32 s6, -1 ; GFX1132_DPP-NEXT: s_mov_b32 s8, exec_lo @@ -7004,8 +6951,8 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s8 ; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132_DPP-NEXT: v_readfirstlane_b32 s2, v8 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v10, v1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v11, v2 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v10, v6 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v11, v7 ; GFX1132_DPP-NEXT: v_readfirstlane_b32 s3, v9 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX1132_DPP-NEXT: v_sub_co_u32 v8, vcc_lo, s2, v10 @@ -7017,173 +6964,166 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace( ; ; GFX1264_DPP-LABEL: sub_i64_varying: ; GFX1264_DPP: ; %bb.0: ; %entry -; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1264_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1264_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1264_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1264_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1264_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1264_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s[0:1] -; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, v2 -; GFX1264_DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] -; GFX1264_DPP-NEXT: v_mov_b32_e32 v6, v2 +; GFX1264_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] +; GFX1264_DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1264_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] +; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1264_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc, v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1264_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v4, vcc -; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc, v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v2, vcc +; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1264_DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1264_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v6, vcc +; GFX1264_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc ; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc, v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1264_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1264_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1264_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v4, vcc -; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1264_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v2, vcc +; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1264_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1264_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1264_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v6, vcc -; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v7, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1264_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1264_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 ; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1264_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1264_DPP-NEXT: v_readlane_b32 s5, v4, 31 -; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1264_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1264_DPP-NEXT: v_mov_b32_e32 v7, s5 -; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v5, vcc -; GFX1264_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v7, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc +; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1264_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1264_DPP-NEXT: v_readlane_b32 s4, v3, 31 -; GFX1264_DPP-NEXT: v_mov_b32_e32 v6, s4 -; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1264_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v5, vcc +; GFX1264_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1264_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc +; GFX1264_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1264_DPP-NEXT: v_readlane_b32 s4, v1, 31 +; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, s4 +; GFX1264_DPP-NEXT: v_readlane_b32 s4, v2, 31 +; GFX1264_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v2, s4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1264_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc ; GFX1264_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1264_DPP-NEXT: s_load_b128 s[0:3], s[2:3], 0x24 ; GFX1264_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[4:5], -1 -; GFX1264_DPP-NEXT: v_mov_b32_dpp v1, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1264_DPP-NEXT: v_readlane_b32 s6, v4, 15 -; GFX1264_DPP-NEXT: v_mov_b32_dpp v2, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1264_DPP-NEXT: v_readlane_b32 s7, v3, 15 -; GFX1264_DPP-NEXT: v_readlane_b32 s8, v4, 31 -; GFX1264_DPP-NEXT: v_readlane_b32 s9, v3, 31 -; GFX1264_DPP-NEXT: v_writelane_b32 v1, s6, 16 -; GFX1264_DPP-NEXT: v_readlane_b32 s6, v4, 63 -; GFX1264_DPP-NEXT: v_writelane_b32 v2, s7, 16 -; GFX1264_DPP-NEXT: v_readlane_b32 s10, v4, 47 -; GFX1264_DPP-NEXT: v_readlane_b32 s11, v3, 47 -; GFX1264_DPP-NEXT: v_readlane_b32 s7, v3, 63 -; GFX1264_DPP-NEXT: v_writelane_b32 v1, s8, 32 -; GFX1264_DPP-NEXT: v_writelane_b32 v2, s9, 32 +; GFX1264_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1264_DPP-NEXT: v_readlane_b32 s6, v2, 15 +; GFX1264_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1264_DPP-NEXT: v_readlane_b32 s7, v1, 15 +; GFX1264_DPP-NEXT: v_readlane_b32 s8, v2, 31 +; GFX1264_DPP-NEXT: v_readlane_b32 s9, v1, 31 +; GFX1264_DPP-NEXT: v_writelane_b32 v4, s6, 16 +; GFX1264_DPP-NEXT: v_readlane_b32 s6, v2, 63 +; GFX1264_DPP-NEXT: v_writelane_b32 v5, s7, 16 +; GFX1264_DPP-NEXT: v_readlane_b32 s10, v2, 47 +; GFX1264_DPP-NEXT: v_readlane_b32 s11, v1, 47 +; GFX1264_DPP-NEXT: v_readlane_b32 s7, v1, 63 +; GFX1264_DPP-NEXT: v_writelane_b32 v4, s8, 32 +; GFX1264_DPP-NEXT: v_writelane_b32 v5, s9, 32 ; GFX1264_DPP-NEXT: s_wait_alu 0xfffe ; GFX1264_DPP-NEXT: s_mov_b64 exec, s[4:5] ; GFX1264_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1264_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 ; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[8:9], -1 ; GFX1264_DPP-NEXT: s_mov_b64 s[4:5], s[6:7] -; GFX1264_DPP-NEXT: v_writelane_b32 v1, s10, 48 -; GFX1264_DPP-NEXT: v_writelane_b32 v2, s11, 48 +; GFX1264_DPP-NEXT: v_writelane_b32 v4, s10, 48 +; GFX1264_DPP-NEXT: v_writelane_b32 v5, s11, 48 ; GFX1264_DPP-NEXT: s_wait_alu 0xfffe ; GFX1264_DPP-NEXT: s_mov_b64 exec, s[8:9] ; GFX1264_DPP-NEXT: s_mov_b32 s6, -1 ; GFX1264_DPP-NEXT: s_mov_b64 s[8:9], exec -; GFX1264_DPP-NEXT: ; implicit-def: $vgpr8_vgpr9 +; GFX1264_DPP-NEXT: ; implicit-def: $vgpr6_vgpr7 ; GFX1264_DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 ; GFX1264_DPP-NEXT: s_cbranch_execz .LBB11_2 ; GFX1264_DPP-NEXT: ; %bb.1: -; GFX1264_DPP-NEXT: v_mov_b32_e32 v9, s5 -; GFX1264_DPP-NEXT: v_mov_b32_e32 v8, s4 +; GFX1264_DPP-NEXT: v_mov_b32_e32 v7, s5 +; GFX1264_DPP-NEXT: v_mov_b32_e32 v6, s4 ; GFX1264_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1264_DPP-NEXT: s_wait_kmcnt 0x0 ; GFX1264_DPP-NEXT: s_mov_b32 s4, s2 ; GFX1264_DPP-NEXT: s_mov_b32 s5, s3 -; GFX1264_DPP-NEXT: buffer_atomic_sub_u64 v[8:9], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1264_DPP-NEXT: buffer_atomic_sub_u64 v[6:7], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX1264_DPP-NEXT: s_wait_loadcnt 0x0 ; GFX1264_DPP-NEXT: global_inv scope:SCOPE_DEV ; GFX1264_DPP-NEXT: .LBB11_2: ; GFX1264_DPP-NEXT: s_wait_alu 0xfffe ; GFX1264_DPP-NEXT: s_or_b64 exec, exec, s[8:9] ; GFX1264_DPP-NEXT: s_wait_kmcnt 0x0 -; GFX1264_DPP-NEXT: v_readfirstlane_b32 s2, v8 -; GFX1264_DPP-NEXT: v_mov_b32_e32 v10, v1 -; GFX1264_DPP-NEXT: v_mov_b32_e32 v11, v2 -; GFX1264_DPP-NEXT: v_readfirstlane_b32 s3, v9 +; GFX1264_DPP-NEXT: v_readfirstlane_b32 s2, v6 +; GFX1264_DPP-NEXT: v_mov_b32_e32 v8, v4 +; GFX1264_DPP-NEXT: v_mov_b32_e32 v9, v5 +; GFX1264_DPP-NEXT: v_readfirstlane_b32 s3, v7 ; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX1264_DPP-NEXT: v_sub_co_u32 v8, vcc, s2, v10 +; GFX1264_DPP-NEXT: v_sub_co_u32 v6, vcc, s2, v8 ; GFX1264_DPP-NEXT: s_mov_b32 s2, s6 -; GFX1264_DPP-NEXT: v_sub_co_ci_u32_e32 v9, vcc, s3, v11, vcc +; GFX1264_DPP-NEXT: v_sub_co_ci_u32_e32 v7, vcc, s3, v9, vcc ; GFX1264_DPP-NEXT: s_mov_b32 s3, 0x31016000 -; GFX1264_DPP-NEXT: buffer_store_b64 v[8:9], off, s[0:3], null +; GFX1264_DPP-NEXT: buffer_store_b64 v[6:7], off, s[0:3], null ; GFX1264_DPP-NEXT: s_endpgm ; ; GFX1232_DPP-LABEL: sub_i64_varying: ; GFX1232_DPP: ; %bb.0: ; %entry -; GFX1232_DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1232_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1232_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1232_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1232_DPP-NEXT: s_or_saveexec_b32 s0, -1 ; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1232_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s0 -; GFX1232_DPP-NEXT: v_mov_b32_e32 v4, v2 -; GFX1232_DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0 -; GFX1232_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1232_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc_lo, v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1232_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s0 +; GFX1232_DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1232_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s0 +; GFX1232_DPP-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v7, 0 +; GFX1232_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1232_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo -; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc_lo, v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1232_DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo +; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1232_DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1232_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v6, vcc_lo +; GFX1232_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo ; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc_lo, v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1232_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1232_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1232_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1232_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo -; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc_lo, v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1232_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo +; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1232_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1232_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1232_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 +; GFX1232_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1232_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 ; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v6, vcc_lo -; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc_lo, v7, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo +; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1232_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1232_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1232_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1232_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1232_DPP-NEXT: s_load_b128 s[0:3], s[2:3], 0x24 ; GFX1232_DPP-NEXT: s_or_saveexec_b32 s6, -1 ; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v5, vcc_lo -; GFX1232_DPP-NEXT: v_readlane_b32 s4, v4, 31 -; GFX1232_DPP-NEXT: v_mov_b32_dpp v1, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1232_DPP-NEXT: v_readlane_b32 s7, v4, 15 -; GFX1232_DPP-NEXT: v_readlane_b32 s8, v3, 15 -; GFX1232_DPP-NEXT: v_readlane_b32 s5, v3, 31 -; GFX1232_DPP-NEXT: v_mov_b32_dpp v2, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1232_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo +; GFX1232_DPP-NEXT: v_readlane_b32 s4, v2, 31 +; GFX1232_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1232_DPP-NEXT: v_readlane_b32 s7, v2, 15 +; GFX1232_DPP-NEXT: v_readlane_b32 s8, v1, 15 +; GFX1232_DPP-NEXT: v_readlane_b32 s5, v1, 31 +; GFX1232_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s6 ; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1232_DPP-NEXT: s_or_saveexec_b32 s6, -1 -; GFX1232_DPP-NEXT: v_writelane_b32 v1, s7, 16 -; GFX1232_DPP-NEXT: v_writelane_b32 v2, s8, 16 +; GFX1232_DPP-NEXT: v_writelane_b32 v6, s7, 16 +; GFX1232_DPP-NEXT: v_writelane_b32 v7, s8, 16 ; GFX1232_DPP-NEXT: s_wait_alu 0xfffe ; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s6 ; GFX1232_DPP-NEXT: s_mov_b32 s6, -1 @@ -7205,8 +7145,8 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX1232_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s8 ; GFX1232_DPP-NEXT: s_wait_kmcnt 0x0 ; GFX1232_DPP-NEXT: v_readfirstlane_b32 s2, v8 -; GFX1232_DPP-NEXT: v_mov_b32_e32 v10, v1 -; GFX1232_DPP-NEXT: v_mov_b32_e32 v11, v2 +; GFX1232_DPP-NEXT: v_mov_b32_e32 v10, v6 +; GFX1232_DPP-NEXT: v_mov_b32_e32 v11, v7 ; GFX1232_DPP-NEXT: v_readfirstlane_b32 s3, v9 ; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX1232_DPP-NEXT: v_sub_co_u32 v8, vcc_lo, s2, v10 diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll index 5924649ebe7350..f7773b4859dc2c 100644 --- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll +++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll @@ -2333,250 +2333,230 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) { ; ; GFX8_DPP-LABEL: add_i64_varying: ; GFX8_DPP: ; %bb.0: ; %entry -; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 -; GFX8_DPP-NEXT: v_mov_b32_e32 v9, 0 -; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v7 +; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v5, exec_lo, 0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v5, exec_hi, v5 ; GFX8_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX8_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 -; GFX8_DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1] -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX8_DPP-NEXT: v_readlane_b32 s5, v1, 63 -; GFX8_DPP-NEXT: v_readlane_b32 s4, v4, 63 -; GFX8_DPP-NEXT: v_mov_b32_dpp v2, v4 wave_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_readlane_b32 s4, v2, 63 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v1 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v2 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX8_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 -; GFX8_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 +; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5 +; GFX8_DPP-NEXT: ; implicit-def: $vgpr5_vgpr6 ; GFX8_DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX8_DPP-NEXT: s_cbranch_execz .LBB6_2 ; GFX8_DPP-NEXT: ; %bb.1: -; GFX8_DPP-NEXT: v_mov_b32_e32 v8, s5 -; GFX8_DPP-NEXT: v_mov_b32_e32 v7, s4 +; GFX8_DPP-NEXT: v_mov_b32_e32 v6, s5 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, s4 ; GFX8_DPP-NEXT: s_mov_b32 m0, -1 -; GFX8_DPP-NEXT: ds_add_rtn_u64 v[7:8], v9, v[7:8] +; GFX8_DPP-NEXT: ds_add_rtn_u64 v[5:6], v7, v[5:6] ; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX8_DPP-NEXT: .LBB6_2: ; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX8_DPP-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24 -; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v8 -; GFX8_DPP-NEXT: v_readfirstlane_b32 s5, v7 -; GFX8_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX8_DPP-NEXT: v_mov_b32_e32 v8, v3 +; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v6 +; GFX8_DPP-NEXT: v_readfirstlane_b32 s5, v5 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 ; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s4 -; GFX8_DPP-NEXT: v_add_u32_e32 v7, vcc, s5, v7 +; GFX8_DPP-NEXT: v_add_u32_e32 v5, vcc, s5, v5 ; GFX8_DPP-NEXT: s_mov_b32 s3, 0xf000 ; GFX8_DPP-NEXT: s_mov_b32 s2, -1 -; GFX8_DPP-NEXT: v_addc_u32_e32 v8, vcc, v0, v8, vcc +; GFX8_DPP-NEXT: v_addc_u32_e32 v6, vcc, v0, v6, vcc ; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX8_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[0:3], 0 +; GFX8_DPP-NEXT: buffer_store_dwordx2 v[5:6], off, s[0:3], 0 ; GFX8_DPP-NEXT: s_endpgm ; ; GFX9_DPP-LABEL: add_i64_varying: ; GFX9_DPP: ; %bb.0: ; %entry -; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 -; GFX9_DPP-NEXT: v_mov_b32_e32 v9, 0 -; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v7 +; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v5, exec_lo, 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v5, exec_hi, v5 ; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 -; GFX9_DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1] -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX9_DPP-NEXT: v_readlane_b32 s5, v1, 63 -; GFX9_DPP-NEXT: v_readlane_b32 s4, v4, 63 -; GFX9_DPP-NEXT: v_mov_b32_dpp v2, v4 wave_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_readlane_b32 s4, v2, 63 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v1 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v2 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 -; GFX9_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 +; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5 +; GFX9_DPP-NEXT: ; implicit-def: $vgpr5_vgpr6 ; GFX9_DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX9_DPP-NEXT: s_cbranch_execz .LBB6_2 ; GFX9_DPP-NEXT: ; %bb.1: -; GFX9_DPP-NEXT: v_mov_b32_e32 v8, s5 -; GFX9_DPP-NEXT: v_mov_b32_e32 v7, s4 -; GFX9_DPP-NEXT: ds_add_rtn_u64 v[7:8], v9, v[7:8] +; GFX9_DPP-NEXT: v_mov_b32_e32 v6, s5 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, s4 +; GFX9_DPP-NEXT: ds_add_rtn_u64 v[5:6], v7, v[5:6] ; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9_DPP-NEXT: .LBB6_2: ; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX9_DPP-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24 -; GFX9_DPP-NEXT: v_readfirstlane_b32 s4, v8 -; GFX9_DPP-NEXT: v_readfirstlane_b32 s5, v7 -; GFX9_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX9_DPP-NEXT: v_mov_b32_e32 v8, v3 +; GFX9_DPP-NEXT: v_readfirstlane_b32 s4, v6 +; GFX9_DPP-NEXT: v_readfirstlane_b32 s5, v5 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 ; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s4 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v7, vcc, s5, v7 +; GFX9_DPP-NEXT: v_add_co_u32_e32 v5, vcc, s5, v5 ; GFX9_DPP-NEXT: s_mov_b32 s3, 0xf000 ; GFX9_DPP-NEXT: s_mov_b32 s2, -1 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v8, vcc, v0, v8, vcc +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v6, vcc, v0, v6, vcc ; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX9_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[0:3], 0 +; GFX9_DPP-NEXT: buffer_store_dwordx2 v[5:6], off, s[0:3], 0 ; GFX9_DPP-NEXT: s_endpgm ; ; GFX1064_DPP-LABEL: add_i64_varying: ; GFX1064_DPP: ; %bb.0: ; %entry ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1064_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] -; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, v1 -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v5, 0, 0, s[0:1] -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v2 +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s[0:1] +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v5 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v4 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v5, v6, vcc -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v7 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v8, vcc -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v5 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v6, vcc -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v7 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v8, vcc -; GFX1064_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1064_DPP-NEXT: v_permlanex16_b32 v8, v4, -1, -1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, 0 +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v2, v1 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v3, v4, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v6 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v4 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v3, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v6 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1064_DPP-NEXT: v_permlanex16_b32 v6, v2, -1, -1 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v5 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v7, vcc -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v3, 31 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v4, 31 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, s4 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s5 +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v1, 31 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v2, 31 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, s4 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, s5 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf ; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v8 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v5 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v7, vcc +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v4, 15 -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v3, 15 -; GFX1064_DPP-NEXT: v_readlane_b32 s6, v4, 31 -; GFX1064_DPP-NEXT: v_readlane_b32 s7, v3, 31 -; GFX1064_DPP-NEXT: v_readlane_b32 s8, v3, 47 -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s4, 16 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s5, 16 -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v4, 63 -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v3, 63 -; GFX1064_DPP-NEXT: v_readlane_b32 s9, v4, 47 -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s6, 32 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s7, 32 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v2, 15 +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v1, 15 +; GFX1064_DPP-NEXT: v_readlane_b32 s6, v2, 31 +; GFX1064_DPP-NEXT: v_readlane_b32 s7, v1, 31 +; GFX1064_DPP-NEXT: v_readlane_b32 s8, v1, 47 +; GFX1064_DPP-NEXT: v_writelane_b32 v8, s4, 16 +; GFX1064_DPP-NEXT: v_writelane_b32 v7, s5, 16 +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v2, 63 +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v1, 63 +; GFX1064_DPP-NEXT: v_readlane_b32 s9, v2, 47 +; GFX1064_DPP-NEXT: v_writelane_b32 v8, s6, 32 +; GFX1064_DPP-NEXT: v_writelane_b32 v7, s7, 32 ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v9, exec_hi, v0 ; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[6:7], -1 ; GFX1064_DPP-NEXT: s_mov_b64 s[0:1], s[4:5] -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s9, 48 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s8, 48 +; GFX1064_DPP-NEXT: v_writelane_b32 v8, s9, 48 +; GFX1064_DPP-NEXT: v_writelane_b32 v7, s8, 48 ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[6:7] ; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9 ; GFX1064_DPP-NEXT: s_mov_b32 s6, -1 @@ -2594,8 +2574,8 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) { ; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1064_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x24 ; GFX1064_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v11, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v12, v2 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v11, v7 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v12, v8 ; GFX1064_DPP-NEXT: v_readfirstlane_b32 s1, v10 ; GFX1064_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1064_DPP-NEXT: v_add_co_u32 v9, vcc, s0, v11 @@ -2608,53 +2588,53 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) { ; GFX1032_DPP: ; %bb.0: ; %entry ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1 ; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s4 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, v1 -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v5, 0, 0, s4 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2 +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s4 +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s4 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v5 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v4 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v5, v6, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v7 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v8, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v5 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v6, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v7 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v8, vcc_lo -; GFX1032_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1032_DPP-NEXT: v_permlanex16_b32 v8, v4, -1, -1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, 0 +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v2, v1 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v3, v4, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v6 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v4 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v3, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v6 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1032_DPP-NEXT: v_permlanex16_b32 v6, v2, -1, -1 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v5 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v7, vcc_lo -; GFX1032_DPP-NEXT: v_readlane_b32 s0, v3, 31 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_readlane_b32 s5, v3, 15 -; GFX1032_DPP-NEXT: v_readlane_b32 s6, v4, 15 -; GFX1032_DPP-NEXT: v_readlane_b32 s1, v4, 31 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo +; GFX1032_DPP-NEXT: v_readlane_b32 s0, v1, 31 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_readlane_b32 s6, v2, 15 +; GFX1032_DPP-NEXT: v_readlane_b32 s1, v2, 31 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_readlane_b32 s5, v1, 15 ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4 ; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0 ; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1032_DPP-NEXT: v_writelane_b32 v2, s6, 16 -; GFX1032_DPP-NEXT: v_writelane_b32 v1, s5, 16 +; GFX1032_DPP-NEXT: v_writelane_b32 v8, s6, 16 +; GFX1032_DPP-NEXT: v_writelane_b32 v7, s5, 16 ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4 ; GFX1032_DPP-NEXT: s_mov_b32 s6, -1 ; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v9 @@ -2672,8 +2652,8 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) { ; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1032_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x24 ; GFX1032_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v11, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v12, v2 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v11, v7 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v12, v8 ; GFX1032_DPP-NEXT: v_readfirstlane_b32 s1, v10 ; GFX1032_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1032_DPP-NEXT: v_add_co_u32 v9, vcc_lo, s0, v11 @@ -2684,165 +2664,158 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) { ; ; GFX1164_DPP-LABEL: add_i64_varying: ; GFX1164_DPP: ; %bb.0: ; %entry -; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, v2 -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc, v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v4, vcc -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc, v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v2, vcc +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v6, vcc +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc, v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v4, vcc -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v2, vcc +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v6, vcc -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v7, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v4, 31 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s5 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v5, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v7, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v3, 31 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, s4 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v5, vcc +; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v1, 31 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s4 +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v2, 31 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v2, s4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v1, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v4, 15 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v3, 15 -; GFX1164_DPP-NEXT: v_readlane_b32 s6, v4, 31 -; GFX1164_DPP-NEXT: v_readlane_b32 s7, v3, 31 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s4, 16 -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v4, 63 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s5, 16 -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v3, 63 -; GFX1164_DPP-NEXT: v_readlane_b32 s8, v4, 47 -; GFX1164_DPP-NEXT: v_readlane_b32 s9, v3, 47 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s6, 32 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s7, 32 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v2, 15 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_readlane_b32 s5, v1, 15 +; GFX1164_DPP-NEXT: v_readlane_b32 s6, v2, 31 +; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 31 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s4, 16 +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v2, 63 +; GFX1164_DPP-NEXT: v_writelane_b32 v6, s5, 16 +; GFX1164_DPP-NEXT: v_readlane_b32 s5, v1, 63 +; GFX1164_DPP-NEXT: v_readlane_b32 s8, v2, 47 +; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 47 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32 +; GFX1164_DPP-NEXT: v_writelane_b32 v6, s7, 32 ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v0 +; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[6:7], -1 ; GFX1164_DPP-NEXT: s_mov_b64 s[0:1], s[4:5] -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s8, 48 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s9, 48 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s8, 48 +; GFX1164_DPP-NEXT: v_writelane_b32 v6, s9, 48 ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7] -; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v8 +; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX1164_DPP-NEXT: s_mov_b32 s6, -1 -; GFX1164_DPP-NEXT: ; implicit-def: $vgpr8_vgpr9 +; GFX1164_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GFX1164_DPP-NEXT: s_cbranch_execz .LBB6_2 ; GFX1164_DPP-NEXT: ; %bb.1: -; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, s1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, s0 -; GFX1164_DPP-NEXT: ds_add_rtn_u64 v[8:9], v0, v[8:9] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, s1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s0 +; GFX1164_DPP-NEXT: ds_add_rtn_u64 v[7:8], v0, v[7:8] ; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164_DPP-NEXT: buffer_gl0_inv ; GFX1164_DPP-NEXT: .LBB6_2: ; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1164_DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x24 -; GFX1164_DPP-NEXT: v_readfirstlane_b32 s0, v8 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v11, v2 -; GFX1164_DPP-NEXT: v_readfirstlane_b32 s1, v9 +; GFX1164_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, v5 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, v6 +; GFX1164_DPP-NEXT: v_readfirstlane_b32 s1, v8 ; GFX1164_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_add_co_u32 v8, vcc, s0, v10 -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v9, vcc, s1, v11, vcc +; GFX1164_DPP-NEXT: v_add_co_u32 v7, vcc, s0, v9 +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v8, vcc, s1, v10, vcc ; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1164_DPP-NEXT: buffer_store_b64 v[8:9], off, s[4:7], 0 +; GFX1164_DPP-NEXT: buffer_store_b64 v[7:8], off, s[4:7], 0 ; GFX1164_DPP-NEXT: s_endpgm ; ; GFX1132_DPP-LABEL: add_i64_varying: ; GFX1132_DPP: ; %bb.0: ; %entry -; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s4 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, v2 -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s4 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc_lo, v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s4 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s4 +; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v7, 0 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc_lo, v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v6, vcc_lo +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc_lo, v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc_lo, v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v6, vcc_lo -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc_lo, v7, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1132_DPP-NEXT: v_readlane_b32 s0, v4, 31 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v1, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_readlane_b32 s5, v4, 15 +; GFX1132_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1132_DPP-NEXT: v_readlane_b32 s0, v2, 31 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_readlane_b32 s5, v2, 15 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v5, vcc_lo +; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1) -; GFX1132_DPP-NEXT: v_readlane_b32 s6, v3, 15 -; GFX1132_DPP-NEXT: v_readlane_b32 s1, v3, 31 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15 +; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 31 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4 ; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v8, exec_lo, 0 ; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1132_DPP-NEXT: v_writelane_b32 v1, s5, 16 -; GFX1132_DPP-NEXT: v_writelane_b32 v2, s6, 16 +; GFX1132_DPP-NEXT: v_writelane_b32 v6, s5, 16 +; GFX1132_DPP-NEXT: v_writelane_b32 v7, s6, 16 ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4 ; GFX1132_DPP-NEXT: s_mov_b32 s6, -1 ; GFX1132_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v8 @@ -2858,8 +2831,8 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) { ; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1132_DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x24 ; GFX1132_DPP-NEXT: v_readfirstlane_b32 s0, v8 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v10, v1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v11, v2 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v10, v6 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v11, v7 ; GFX1132_DPP-NEXT: v_readfirstlane_b32 s1, v9 ; GFX1132_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) @@ -3125,137 +3098,119 @@ define amdgpu_kernel void @add_i64_varying_nouse() { ; ; GFX8_DPP-LABEL: add_i64_varying_nouse: ; GFX8_DPP: ; %bb.0: ; %entry -; GFX8_DPP-NEXT: v_mov_b32_e32 v7, 0 -; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v8, exec_lo, 0 -; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v8 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0 +; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v6 ; GFX8_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX8_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 -; GFX8_DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1] -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: v_mov_b32_dpp v2, v4 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: s_nop 0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4 ; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v4, v2 ; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc ; GFX8_DPP-NEXT: v_readlane_b32 s3, v1, 63 ; GFX8_DPP-NEXT: v_readlane_b32 s2, v2, 63 ; GFX8_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX8_DPP-NEXT: s_mov_b64 s[0:1], s[2:3] -; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v8 +; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6 ; GFX8_DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc ; GFX8_DPP-NEXT: s_cbranch_execz .LBB7_2 ; GFX8_DPP-NEXT: ; %bb.1: -; GFX8_DPP-NEXT: v_mov_b32_e32 v9, s1 -; GFX8_DPP-NEXT: v_mov_b32_e32 v8, s0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v7, s1 +; GFX8_DPP-NEXT: v_mov_b32_e32 v6, s0 ; GFX8_DPP-NEXT: s_mov_b32 m0, -1 -; GFX8_DPP-NEXT: ds_add_u64 v7, v[8:9] +; GFX8_DPP-NEXT: ds_add_u64 v5, v[6:7] ; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX8_DPP-NEXT: .LBB7_2: ; GFX8_DPP-NEXT: s_endpgm ; ; GFX9_DPP-LABEL: add_i64_varying_nouse: ; GFX9_DPP: ; %bb.0: ; %entry -; GFX9_DPP-NEXT: v_mov_b32_e32 v7, 0 -; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v8, exec_lo, 0 -; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v8 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0 +; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v6 ; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 -; GFX9_DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1] -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: v_mov_b32_dpp v2, v4 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: s_nop 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 ; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2 ; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc ; GFX9_DPP-NEXT: v_readlane_b32 s3, v1, 63 ; GFX9_DPP-NEXT: v_readlane_b32 s2, v2, 63 ; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX9_DPP-NEXT: s_mov_b64 s[0:1], s[2:3] -; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v8 +; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6 ; GFX9_DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc ; GFX9_DPP-NEXT: s_cbranch_execz .LBB7_2 ; GFX9_DPP-NEXT: ; %bb.1: -; GFX9_DPP-NEXT: v_mov_b32_e32 v9, s1 -; GFX9_DPP-NEXT: v_mov_b32_e32 v8, s0 -; GFX9_DPP-NEXT: ds_add_u64 v7, v[8:9] +; GFX9_DPP-NEXT: v_mov_b32_e32 v7, s1 +; GFX9_DPP-NEXT: v_mov_b32_e32 v6, s0 +; GFX9_DPP-NEXT: ds_add_u64 v5, v[6:7] ; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9_DPP-NEXT: .LBB7_2: ; GFX9_DPP-NEXT: s_endpgm @@ -3264,31 +3219,31 @@ define amdgpu_kernel void @add_i64_varying_nouse() { ; GFX1064_DPP: ; %bb.0: ; %entry ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1064_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] -; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, v1 -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v5, 0, 0, s[0:1] -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v2 +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s[0:1] +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v4 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v5, v6, vcc -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v3 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v4 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v7 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v8, vcc -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v5 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v6, vcc -; GFX1064_DPP-NEXT: v_mov_b32_dpp v1, v3 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v3, v1 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v4, v2, vcc +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v2, v1 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v3, v4, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v6 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v2 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v4 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v3, vcc +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v6 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc ; GFX1064_DPP-NEXT: v_permlanex16_b32 v3, v1, 0, 0 ; GFX1064_DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0 ; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3 @@ -3301,17 +3256,17 @@ define amdgpu_kernel void @add_i64_varying_nouse() { ; GFX1064_DPP-NEXT: v_readlane_b32 s4, v1, 32 ; GFX1064_DPP-NEXT: v_readlane_b32 s5, v2, 32 ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v9, exec_hi, v0 +; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1064_DPP-NEXT: s_add_u32 s0, s3, s4 ; GFX1064_DPP-NEXT: s_addc_u32 s1, s2, s5 -; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9 +; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc ; GFX1064_DPP-NEXT: s_cbranch_execz .LBB7_2 ; GFX1064_DPP-NEXT: ; %bb.1: -; GFX1064_DPP-NEXT: v_mov_b32_e32 v10, s1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, s0 -; GFX1064_DPP-NEXT: ds_add_u64 v0, v[9:10] +; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, s0 +; GFX1064_DPP-NEXT: ds_add_u64 v0, v[7:8] ; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064_DPP-NEXT: buffer_gl0_inv ; GFX1064_DPP-NEXT: .LBB7_2: @@ -3321,45 +3276,45 @@ define amdgpu_kernel void @add_i64_varying_nouse() { ; GFX1032_DPP: ; %bb.0: ; %entry ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1 ; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s0 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, v1 -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v5, 0, 0, s0 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2 +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0 +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v4 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v5, v6, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v3 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v4 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v7 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v8, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v5 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v6, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_dpp v1, v3 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v3, v1 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v4, v2, vcc_lo +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v2, v1 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v3, v4, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v1 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v6 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v2 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v4 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v3, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v1 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v6 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo ; GFX1032_DPP-NEXT: v_permlanex16_b32 v3, v1, 0, 0 ; GFX1032_DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0 ; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3 ; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v4, vcc_lo ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, v1 -; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v11, exec_lo, 0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1 +; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0 ; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v10, v2 -; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v11 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2 +; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v9 ; GFX1032_DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX1032_DPP-NEXT: s_cbranch_execz .LBB7_2 ; GFX1032_DPP-NEXT: ; %bb.1: -; GFX1032_DPP-NEXT: ds_add_u64 v0, v[9:10] +; GFX1032_DPP-NEXT: ds_add_u64 v0, v[7:8] ; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032_DPP-NEXT: buffer_gl0_inv ; GFX1032_DPP-NEXT: .LBB7_2: @@ -3367,44 +3322,41 @@ define amdgpu_kernel void @add_i64_varying_nouse() { ; ; GFX1164_DPP-LABEL: add_i64_varying_nouse: ; GFX1164_DPP: ; %bb.0: ; %entry -; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v2 -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v4, v4 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v4, v4 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v2 +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v2, vcc +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v1 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v5, vcc -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v4, v4 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v4, v4 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v2, vcc -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_permlanex16_b32 v2, v3, 0, 0 -; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, 0, 0 +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_permlanex16_b32 v3, v2, 0, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_add_co_u32 v2, vcc, v3, v2 ; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc +; GFX1164_DPP-NEXT: v_add_co_u32 v2, vcc, v2, v3 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, 0, 0 ; GFX1164_DPP-NEXT: v_permlane64_b32 v3, v2 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc ; GFX1164_DPP-NEXT: v_permlane64_b32 v4, v1 ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_2) @@ -3413,16 +3365,16 @@ define amdgpu_kernel void @add_i64_varying_nouse() { ; GFX1164_DPP-NEXT: v_add_co_u32 v2, vcc, v2, v3 ; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v1, v4, vcc ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v0 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, v3 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v3 ; GFX1164_DPP-NEXT: s_mov_b64 s[0:1], exec ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v8 +; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v7 ; GFX1164_DPP-NEXT: s_cbranch_execz .LBB7_2 ; GFX1164_DPP-NEXT: ; %bb.1: -; GFX1164_DPP-NEXT: ds_add_u64 v0, v[6:7] +; GFX1164_DPP-NEXT: ds_add_u64 v0, v[5:6] ; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164_DPP-NEXT: buffer_gl0_inv ; GFX1164_DPP-NEXT: .LBB7_2: @@ -3430,53 +3382,50 @@ define amdgpu_kernel void @add_i64_varying_nouse() { ; ; GFX1132_DPP-LABEL: add_i64_varying_nouse: ; GFX1132_DPP: ; %bb.0: ; %entry -; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s0 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, v2 -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s0 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc_lo, v4, v4 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc_lo, v4, v4 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, v2 +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v1 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v5, vcc_lo -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc_lo, v4, v4 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v4, v4 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v3, 0, 0 -; GFX1132_DPP-NEXT: v_permlanex16_b32 v4, v1, 0, 0 +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v3, v3 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_add_co_u32 v2, vcc_lo, v3, v2 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_permlanex16_b32 v3, v2, 0, 0 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo +; GFX1132_DPP-NEXT: v_add_co_u32 v2, vcc_lo, v2, v3 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132_DPP-NEXT: v_permlanex16_b32 v4, v1, 0, 0 ; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v1, v4, vcc_lo ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v8, exec_lo, 0 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v7, v3 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1132_DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v5, v2 +; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, v3 ; GFX1132_DPP-NEXT: s_mov_b32 s0, exec_lo ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v8 +; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v7 ; GFX1132_DPP-NEXT: s_cbranch_execz .LBB7_2 ; GFX1132_DPP-NEXT: ; %bb.1: -; GFX1132_DPP-NEXT: ds_add_u64 v0, v[6:7] +; GFX1132_DPP-NEXT: ds_add_u64 v0, v[5:6] ; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132_DPP-NEXT: buffer_gl0_inv ; GFX1132_DPP-NEXT: .LBB7_2: @@ -5835,250 +5784,230 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) { ; ; GFX8_DPP-LABEL: sub_i64_varying: ; GFX8_DPP: ; %bb.0: ; %entry -; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 -; GFX8_DPP-NEXT: v_mov_b32_e32 v9, 0 -; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v7 +; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v5, exec_lo, 0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v5, exec_hi, v5 ; GFX8_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX8_DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 -; GFX8_DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1] -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX8_DPP-NEXT: v_add_u32_e32 v4, vcc, v4, v5 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v6, vcc ; GFX8_DPP-NEXT: v_readlane_b32 s5, v1, 63 -; GFX8_DPP-NEXT: v_readlane_b32 s4, v4, 63 -; GFX8_DPP-NEXT: v_mov_b32_dpp v2, v4 wave_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_readlane_b32 s4, v2, 63 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v1 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v2 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX8_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 -; GFX8_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 +; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5 +; GFX8_DPP-NEXT: ; implicit-def: $vgpr5_vgpr6 ; GFX8_DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX8_DPP-NEXT: s_cbranch_execz .LBB14_2 ; GFX8_DPP-NEXT: ; %bb.1: -; GFX8_DPP-NEXT: v_mov_b32_e32 v8, s5 -; GFX8_DPP-NEXT: v_mov_b32_e32 v7, s4 +; GFX8_DPP-NEXT: v_mov_b32_e32 v6, s5 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, s4 ; GFX8_DPP-NEXT: s_mov_b32 m0, -1 -; GFX8_DPP-NEXT: ds_sub_rtn_u64 v[7:8], v9, v[7:8] +; GFX8_DPP-NEXT: ds_sub_rtn_u64 v[5:6], v7, v[5:6] ; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX8_DPP-NEXT: .LBB14_2: ; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX8_DPP-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24 -; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v8 -; GFX8_DPP-NEXT: v_readfirstlane_b32 s5, v7 -; GFX8_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX8_DPP-NEXT: v_mov_b32_e32 v8, v3 +; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v6 +; GFX8_DPP-NEXT: v_readfirstlane_b32 s5, v5 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 ; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s4 -; GFX8_DPP-NEXT: v_sub_u32_e32 v7, vcc, s5, v7 +; GFX8_DPP-NEXT: v_sub_u32_e32 v5, vcc, s5, v5 ; GFX8_DPP-NEXT: s_mov_b32 s3, 0xf000 ; GFX8_DPP-NEXT: s_mov_b32 s2, -1 -; GFX8_DPP-NEXT: v_subb_u32_e32 v8, vcc, v0, v8, vcc +; GFX8_DPP-NEXT: v_subb_u32_e32 v6, vcc, v0, v6, vcc ; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX8_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[0:3], 0 +; GFX8_DPP-NEXT: buffer_store_dwordx2 v[5:6], off, s[0:3], 0 ; GFX8_DPP-NEXT: s_endpgm ; ; GFX9_DPP-LABEL: sub_i64_varying: ; GFX9_DPP: ; %bb.0: ; %entry -; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 -; GFX9_DPP-NEXT: v_mov_b32_e32 v9, 0 -; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v7 +; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v5, exec_lo, 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v5, exec_hi, v5 ; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 -; GFX9_DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1] -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v3 -; GFX9_DPP-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v6, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX9_DPP-NEXT: v_readlane_b32 s5, v1, 63 -; GFX9_DPP-NEXT: v_readlane_b32 s4, v4, 63 -; GFX9_DPP-NEXT: v_mov_b32_dpp v2, v4 wave_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_readlane_b32 s4, v2, 63 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v1 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v2 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 -; GFX9_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 +; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5 +; GFX9_DPP-NEXT: ; implicit-def: $vgpr5_vgpr6 ; GFX9_DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX9_DPP-NEXT: s_cbranch_execz .LBB14_2 ; GFX9_DPP-NEXT: ; %bb.1: -; GFX9_DPP-NEXT: v_mov_b32_e32 v8, s5 -; GFX9_DPP-NEXT: v_mov_b32_e32 v7, s4 -; GFX9_DPP-NEXT: ds_sub_rtn_u64 v[7:8], v9, v[7:8] +; GFX9_DPP-NEXT: v_mov_b32_e32 v6, s5 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, s4 +; GFX9_DPP-NEXT: ds_sub_rtn_u64 v[5:6], v7, v[5:6] ; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9_DPP-NEXT: .LBB14_2: ; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX9_DPP-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24 -; GFX9_DPP-NEXT: v_readfirstlane_b32 s4, v8 -; GFX9_DPP-NEXT: v_readfirstlane_b32 s5, v7 -; GFX9_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX9_DPP-NEXT: v_mov_b32_e32 v8, v3 +; GFX9_DPP-NEXT: v_readfirstlane_b32 s4, v6 +; GFX9_DPP-NEXT: v_readfirstlane_b32 s5, v5 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 ; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s4 -; GFX9_DPP-NEXT: v_sub_co_u32_e32 v7, vcc, s5, v7 +; GFX9_DPP-NEXT: v_sub_co_u32_e32 v5, vcc, s5, v5 ; GFX9_DPP-NEXT: s_mov_b32 s3, 0xf000 ; GFX9_DPP-NEXT: s_mov_b32 s2, -1 -; GFX9_DPP-NEXT: v_subb_co_u32_e32 v8, vcc, v0, v8, vcc +; GFX9_DPP-NEXT: v_subb_co_u32_e32 v6, vcc, v0, v6, vcc ; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX9_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[0:3], 0 +; GFX9_DPP-NEXT: buffer_store_dwordx2 v[5:6], off, s[0:3], 0 ; GFX9_DPP-NEXT: s_endpgm ; ; GFX1064_DPP-LABEL: sub_i64_varying: ; GFX1064_DPP: ; %bb.0: ; %entry ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1064_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] -; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, v1 -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v5, 0, 0, s[0:1] -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v2 +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s[0:1] +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v5 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v4 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v5, v6, vcc -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v7 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v8, vcc -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v5 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v6, vcc -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v7 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v8, vcc -; GFX1064_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1064_DPP-NEXT: v_permlanex16_b32 v8, v4, -1, -1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, 0 +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v2, v1 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v3, v4, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v6 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v4 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v3, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v6 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1064_DPP-NEXT: v_permlanex16_b32 v6, v2, -1, -1 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v5 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v7, vcc -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v3, 31 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v4, 31 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, s4 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s5 +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v1, 31 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v2, 31 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, s4 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, s5 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf ; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v8 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1064_DPP-NEXT: v_add_co_u32 v3, vcc, v3, v5 -; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc, v4, v7, vcc +; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3 +; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v4, 15 -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v3, 15 -; GFX1064_DPP-NEXT: v_readlane_b32 s6, v4, 31 -; GFX1064_DPP-NEXT: v_readlane_b32 s7, v3, 31 -; GFX1064_DPP-NEXT: v_readlane_b32 s8, v3, 47 -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s4, 16 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s5, 16 -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v4, 63 -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v3, 63 -; GFX1064_DPP-NEXT: v_readlane_b32 s9, v4, 47 -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s6, 32 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s7, 32 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v2, 15 +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v1, 15 +; GFX1064_DPP-NEXT: v_readlane_b32 s6, v2, 31 +; GFX1064_DPP-NEXT: v_readlane_b32 s7, v1, 31 +; GFX1064_DPP-NEXT: v_readlane_b32 s8, v1, 47 +; GFX1064_DPP-NEXT: v_writelane_b32 v8, s4, 16 +; GFX1064_DPP-NEXT: v_writelane_b32 v7, s5, 16 +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v2, 63 +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v1, 63 +; GFX1064_DPP-NEXT: v_readlane_b32 s9, v2, 47 +; GFX1064_DPP-NEXT: v_writelane_b32 v8, s6, 32 +; GFX1064_DPP-NEXT: v_writelane_b32 v7, s7, 32 ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v9, exec_hi, v0 ; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[6:7], -1 ; GFX1064_DPP-NEXT: s_mov_b64 s[0:1], s[4:5] -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s9, 48 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s8, 48 +; GFX1064_DPP-NEXT: v_writelane_b32 v8, s9, 48 +; GFX1064_DPP-NEXT: v_writelane_b32 v7, s8, 48 ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[6:7] ; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9 ; GFX1064_DPP-NEXT: s_mov_b32 s6, -1 @@ -6096,8 +6025,8 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) { ; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1064_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x24 ; GFX1064_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v11, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v12, v2 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v11, v7 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v12, v8 ; GFX1064_DPP-NEXT: v_readfirstlane_b32 s1, v10 ; GFX1064_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1064_DPP-NEXT: v_sub_co_u32 v9, vcc, s0, v11 @@ -6110,53 +6039,53 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) { ; GFX1032_DPP: ; %bb.0: ; %entry ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1 ; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s4 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, v1 -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v5, 0, 0, s4 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2 +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s4 +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s4 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v5 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v4 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v5, v6, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v7 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v8, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v5 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v6, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v7 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v2 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v8, vcc_lo -; GFX1032_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1032_DPP-NEXT: v_permlanex16_b32 v8, v4, -1, -1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, 0 +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v2, v1 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v3, v4, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v6 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v4 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v3, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v6 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1032_DPP-NEXT: v_permlanex16_b32 v6, v2, -1, -1 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1032_DPP-NEXT: v_add_co_u32 v3, vcc_lo, v3, v5 -; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v7, vcc_lo -; GFX1032_DPP-NEXT: v_readlane_b32 s0, v3, 31 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_readlane_b32 s5, v3, 15 -; GFX1032_DPP-NEXT: v_readlane_b32 s6, v4, 15 -; GFX1032_DPP-NEXT: v_readlane_b32 s1, v4, 31 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3 +; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo +; GFX1032_DPP-NEXT: v_readlane_b32 s0, v1, 31 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_readlane_b32 s6, v2, 15 +; GFX1032_DPP-NEXT: v_readlane_b32 s1, v2, 31 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_readlane_b32 s5, v1, 15 ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4 ; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0 ; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1032_DPP-NEXT: v_writelane_b32 v2, s6, 16 -; GFX1032_DPP-NEXT: v_writelane_b32 v1, s5, 16 +; GFX1032_DPP-NEXT: v_writelane_b32 v8, s6, 16 +; GFX1032_DPP-NEXT: v_writelane_b32 v7, s5, 16 ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4 ; GFX1032_DPP-NEXT: s_mov_b32 s6, -1 ; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v9 @@ -6174,8 +6103,8 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) { ; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1032_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x24 ; GFX1032_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v11, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v12, v2 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v11, v7 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v12, v8 ; GFX1032_DPP-NEXT: v_readfirstlane_b32 s1, v10 ; GFX1032_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1032_DPP-NEXT: v_sub_co_u32 v9, vcc_lo, s0, v11 @@ -6186,165 +6115,158 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) { ; ; GFX1164_DPP-LABEL: sub_i64_varying: ; GFX1164_DPP: ; %bb.0: ; %entry -; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, v2 -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc, v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v4, vcc -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc, v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v2, vcc +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v6, vcc +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc, v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v4, vcc -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v2, vcc +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v6, vcc -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v7, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v4, 31 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s5 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v5, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v2 -; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc, v7, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v4, vcc +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v3, 31 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, s4 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc, v3, v5, vcc +; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v1, 31 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s4 +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v2, 31 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v2, s4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc, v1, v3, vcc ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v1, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v4, 15 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v3, 15 -; GFX1164_DPP-NEXT: v_readlane_b32 s6, v4, 31 -; GFX1164_DPP-NEXT: v_readlane_b32 s7, v3, 31 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s4, 16 -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v4, 63 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s5, 16 -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v3, 63 -; GFX1164_DPP-NEXT: v_readlane_b32 s8, v4, 47 -; GFX1164_DPP-NEXT: v_readlane_b32 s9, v3, 47 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s6, 32 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s7, 32 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v2, 15 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_readlane_b32 s5, v1, 15 +; GFX1164_DPP-NEXT: v_readlane_b32 s6, v2, 31 +; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 31 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s4, 16 +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v2, 63 +; GFX1164_DPP-NEXT: v_writelane_b32 v6, s5, 16 +; GFX1164_DPP-NEXT: v_readlane_b32 s5, v1, 63 +; GFX1164_DPP-NEXT: v_readlane_b32 s8, v2, 47 +; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 47 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32 +; GFX1164_DPP-NEXT: v_writelane_b32 v6, s7, 32 ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v0 +; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[6:7], -1 ; GFX1164_DPP-NEXT: s_mov_b64 s[0:1], s[4:5] -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s8, 48 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s9, 48 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s8, 48 +; GFX1164_DPP-NEXT: v_writelane_b32 v6, s9, 48 ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7] -; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v8 +; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX1164_DPP-NEXT: s_mov_b32 s6, -1 -; GFX1164_DPP-NEXT: ; implicit-def: $vgpr8_vgpr9 +; GFX1164_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GFX1164_DPP-NEXT: s_cbranch_execz .LBB14_2 ; GFX1164_DPP-NEXT: ; %bb.1: -; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, s1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, s0 -; GFX1164_DPP-NEXT: ds_sub_rtn_u64 v[8:9], v0, v[8:9] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, s1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s0 +; GFX1164_DPP-NEXT: ds_sub_rtn_u64 v[7:8], v0, v[7:8] ; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164_DPP-NEXT: buffer_gl0_inv ; GFX1164_DPP-NEXT: .LBB14_2: ; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1164_DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x24 -; GFX1164_DPP-NEXT: v_readfirstlane_b32 s0, v8 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v11, v2 -; GFX1164_DPP-NEXT: v_readfirstlane_b32 s1, v9 +; GFX1164_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, v5 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, v6 +; GFX1164_DPP-NEXT: v_readfirstlane_b32 s1, v8 ; GFX1164_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_sub_co_u32 v8, vcc, s0, v10 -; GFX1164_DPP-NEXT: v_sub_co_ci_u32_e32 v9, vcc, s1, v11, vcc +; GFX1164_DPP-NEXT: v_sub_co_u32 v7, vcc, s0, v9 +; GFX1164_DPP-NEXT: v_sub_co_ci_u32_e32 v8, vcc, s1, v10, vcc ; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1164_DPP-NEXT: buffer_store_b64 v[8:9], off, s[4:7], 0 +; GFX1164_DPP-NEXT: buffer_store_b64 v[7:8], off, s[4:7], 0 ; GFX1164_DPP-NEXT: s_endpgm ; ; GFX1132_DPP-LABEL: sub_i64_varying: ; GFX1132_DPP: ; %bb.0: ; %entry -; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s4 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, v2 -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s4 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc_lo, v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s4 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s4 +; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v7, 0 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc_lo, v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v6, vcc_lo +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v5, vcc_lo, v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc_lo, v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v5, v2 +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v2, vcc_lo +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v6, vcc_lo -; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v4, vcc_lo, v7, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo +; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_permlanex16_b32 v6, v3, -1, -1 -; GFX1132_DPP-NEXT: v_readlane_b32 s0, v4, 31 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v1, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_readlane_b32 s5, v4, 15 +; GFX1132_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1 +; GFX1132_DPP-NEXT: v_readlane_b32 s0, v2, 31 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_readlane_b32 s5, v2, 15 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v5, vcc_lo +; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1132_DPP-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1) -; GFX1132_DPP-NEXT: v_readlane_b32 s6, v3, 15 -; GFX1132_DPP-NEXT: v_readlane_b32 s1, v3, 31 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15 +; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 31 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4 ; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v8, exec_lo, 0 ; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1132_DPP-NEXT: v_writelane_b32 v1, s5, 16 -; GFX1132_DPP-NEXT: v_writelane_b32 v2, s6, 16 +; GFX1132_DPP-NEXT: v_writelane_b32 v6, s5, 16 +; GFX1132_DPP-NEXT: v_writelane_b32 v7, s6, 16 ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4 ; GFX1132_DPP-NEXT: s_mov_b32 s6, -1 ; GFX1132_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v8 @@ -6360,8 +6282,8 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) { ; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1132_DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x24 ; GFX1132_DPP-NEXT: v_readfirstlane_b32 s0, v8 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v10, v1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v11, v2 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v10, v6 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v11, v7 ; GFX1132_DPP-NEXT: v_readfirstlane_b32 s1, v9 ; GFX1132_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) @@ -7339,30 +7261,30 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) { ; GFX8_DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v5, exec_hi, v5 ; GFX8_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, -1, 0, s[0:1] -; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, -1, v0, s[0:1] -; GFX8_DPP-NEXT: v_mov_b32_e32 v3, -1 -; GFX8_DPP-NEXT: v_and_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: s_mov_b32 s6, -1 -; GFX8_DPP-NEXT: v_and_b32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_e32 v4, -1 -; GFX8_DPP-NEXT: v_and_b32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_cndmask_b32_e64 v3, -1, 0, s[0:1] +; GFX8_DPP-NEXT: v_cndmask_b32_e64 v4, -1, v0, s[0:1] +; GFX8_DPP-NEXT: v_mov_b32_e32 v1, -1 +; GFX8_DPP-NEXT: v_and_b32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_and_b32_dpp v4, v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_e32 v2, -1 +; GFX8_DPP-NEXT: v_and_b32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_and_b32_dpp v4, v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_and_b32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_and_b32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_and_b32_dpp v4, v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_and_b32_dpp v1, v1, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_and_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_and_b32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_and_b32_dpp v4, v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_and_b32_dpp v1, v1, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_and_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_readlane_b32 s5, v1, 63 -; GFX8_DPP-NEXT: v_readlane_b32 s4, v2, 63 -; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v2 wave_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v1 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_and_b32_dpp v3, v3, v3 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_and_b32_dpp v4, v4, v4 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: s_nop 0 +; GFX8_DPP-NEXT: v_and_b32_dpp v3, v3, v3 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_and_b32_dpp v4, v4, v4 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_readlane_b32 s5, v3, 63 +; GFX8_DPP-NEXT: v_readlane_b32 s4, v4, 63 +; GFX8_DPP-NEXT: v_mov_b32_dpp v2, v3 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v1, v4 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX8_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5 ; GFX8_DPP-NEXT: ; implicit-def: $vgpr5_vgpr6 @@ -7376,16 +7298,17 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) { ; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX8_DPP-NEXT: .LBB16_2: ; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[0:1] -; GFX8_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x24 -; GFX8_DPP-NEXT: v_readfirstlane_b32 s0, v6 -; GFX8_DPP-NEXT: v_readfirstlane_b32 s1, v5 -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 -; GFX8_DPP-NEXT: s_mov_b32 s7, 0xf000 -; GFX8_DPP-NEXT: v_and_b32_e32 v6, s0, v6 -; GFX8_DPP-NEXT: v_and_b32_e32 v5, s1, v5 +; GFX8_DPP-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24 +; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v6 +; GFX8_DPP-NEXT: v_readfirstlane_b32 s5, v5 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v1 +; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v2 +; GFX8_DPP-NEXT: s_mov_b32 s3, 0xf000 +; GFX8_DPP-NEXT: s_mov_b32 s2, -1 +; GFX8_DPP-NEXT: v_and_b32_e32 v6, s4, v6 +; GFX8_DPP-NEXT: v_and_b32_e32 v5, s5, v5 ; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX8_DPP-NEXT: buffer_store_dwordx2 v[5:6], off, s[4:7], 0 +; GFX8_DPP-NEXT: buffer_store_dwordx2 v[5:6], off, s[0:3], 0 ; GFX8_DPP-NEXT: s_endpgm ; ; GFX9_DPP-LABEL: and_i64_varying: @@ -7394,30 +7317,30 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) { ; GFX9_DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v5, exec_hi, v5 ; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, -1, 0, s[0:1] -; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, -1, v0, s[0:1] -; GFX9_DPP-NEXT: v_mov_b32_e32 v3, -1 -; GFX9_DPP-NEXT: v_and_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: s_mov_b32 s6, -1 -; GFX9_DPP-NEXT: v_and_b32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_e32 v4, -1 -; GFX9_DPP-NEXT: v_and_b32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_cndmask_b32_e64 v3, -1, 0, s[0:1] +; GFX9_DPP-NEXT: v_cndmask_b32_e64 v4, -1, v0, s[0:1] +; GFX9_DPP-NEXT: v_mov_b32_e32 v1, -1 +; GFX9_DPP-NEXT: v_and_b32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_and_b32_dpp v4, v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_e32 v2, -1 +; GFX9_DPP-NEXT: v_and_b32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_and_b32_dpp v4, v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_and_b32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_and_b32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_and_b32_dpp v4, v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_and_b32_dpp v1, v1, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_and_b32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_and_b32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_and_b32_dpp v4, v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_and_b32_dpp v1, v1, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_and_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_readlane_b32 s5, v1, 63 -; GFX9_DPP-NEXT: v_readlane_b32 s4, v2, 63 -; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v2 wave_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v1 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_and_b32_dpp v3, v3, v3 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_and_b32_dpp v4, v4, v4 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: s_nop 0 +; GFX9_DPP-NEXT: v_and_b32_dpp v3, v3, v3 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_and_b32_dpp v4, v4, v4 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_readlane_b32 s5, v3, 63 +; GFX9_DPP-NEXT: v_readlane_b32 s4, v4, 63 +; GFX9_DPP-NEXT: v_mov_b32_dpp v2, v3 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v1, v4 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5 ; GFX9_DPP-NEXT: ; implicit-def: $vgpr5_vgpr6 @@ -7430,16 +7353,17 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) { ; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9_DPP-NEXT: .LBB16_2: ; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[0:1] -; GFX9_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x24 -; GFX9_DPP-NEXT: v_readfirstlane_b32 s0, v6 -; GFX9_DPP-NEXT: v_readfirstlane_b32 s1, v5 -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 -; GFX9_DPP-NEXT: s_mov_b32 s7, 0xf000 -; GFX9_DPP-NEXT: v_and_b32_e32 v6, s0, v6 -; GFX9_DPP-NEXT: v_and_b32_e32 v5, s1, v5 +; GFX9_DPP-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24 +; GFX9_DPP-NEXT: v_readfirstlane_b32 s4, v6 +; GFX9_DPP-NEXT: v_readfirstlane_b32 s5, v5 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v1 +; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v2 +; GFX9_DPP-NEXT: s_mov_b32 s3, 0xf000 +; GFX9_DPP-NEXT: s_mov_b32 s2, -1 +; GFX9_DPP-NEXT: v_and_b32_e32 v6, s4, v6 +; GFX9_DPP-NEXT: v_and_b32_e32 v5, s5, v5 ; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX9_DPP-NEXT: buffer_store_dwordx2 v[5:6], off, s[4:7], 0 +; GFX9_DPP-NEXT: buffer_store_dwordx2 v[5:6], off, s[0:3], 0 ; GFX9_DPP-NEXT: s_endpgm ; ; GFX1064_DPP-LABEL: and_i64_varying: @@ -7447,8 +7371,8 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) { ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v1, -1, 0, s[0:1] ; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v2, -1, v0, s[0:1] -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, -1 ; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, -1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, -1 ; GFX1064_DPP-NEXT: v_and_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1064_DPP-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1064_DPP-NEXT: v_and_b32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf @@ -7523,8 +7447,8 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) { ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1 ; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, -1, 0, s4 ; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, -1, v0, s4 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, -1 ; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, -1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, -1 ; GFX1032_DPP-NEXT: v_and_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1032_DPP-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1032_DPP-NEXT: v_and_b32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf @@ -7583,8 +7507,8 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) { ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, -1, 0, s[0:1] ; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v2, -1, v0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, -1 ; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, -1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1164_DPP-NEXT: v_and_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1164_DPP-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf @@ -7672,9 +7596,8 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) { ; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, -1, 0, s4 ; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v2, -1, v0, s4 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v5, -1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, -1 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1132_DPP-NEXT: v_dual_mov_b32 v6, -1 :: v_dual_mov_b32 v5, -1 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1132_DPP-NEXT: v_and_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: v_and_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) @@ -8702,10 +8625,10 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) { ; GFX8_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] ; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] -; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX8_DPP-NEXT: v_or_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX8_DPP-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX8_DPP-NEXT: v_or_b32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX8_DPP-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX8_DPP-NEXT: s_nop 0 @@ -8722,8 +8645,8 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) { ; GFX8_DPP-NEXT: v_or_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf ; GFX8_DPP-NEXT: v_readlane_b32 s5, v1, 63 ; GFX8_DPP-NEXT: v_readlane_b32 s4, v2, 63 -; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v2 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v1 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v2 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX8_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5 ; GFX8_DPP-NEXT: ; implicit-def: $vgpr5_vgpr6 @@ -8758,10 +8681,10 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) { ; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] ; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] -; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX9_DPP-NEXT: v_or_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX9_DPP-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX9_DPP-NEXT: v_or_b32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX9_DPP-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX9_DPP-NEXT: s_nop 0 @@ -8778,8 +8701,8 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) { ; GFX9_DPP-NEXT: v_or_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf ; GFX9_DPP-NEXT: v_readlane_b32 s5, v1, 63 ; GFX9_DPP-NEXT: v_readlane_b32 s4, v2, 63 -; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v2 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v1 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v2 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5 ; GFX9_DPP-NEXT: ; implicit-def: $vgpr5_vgpr6 @@ -8810,8 +8733,8 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) { ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] ; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX1064_DPP-NEXT: v_or_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1064_DPP-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1064_DPP-NEXT: v_or_b32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 @@ -8886,8 +8809,8 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) { ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1 ; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s4 ; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s4 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX1032_DPP-NEXT: v_or_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1032_DPP-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1032_DPP-NEXT: v_or_b32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 @@ -8946,8 +8869,8 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) { ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] ; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1164_DPP-NEXT: v_or_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1164_DPP-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 @@ -9035,9 +8958,8 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) { ; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s4 ; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s4 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v5, 0 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, 0 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1132_DPP-NEXT: v_dual_mov_b32 v6, 0 :: v_dual_mov_b32 v5, 0 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1132_DPP-NEXT: v_or_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1132_DPP-NEXT: v_or_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) @@ -10065,10 +9987,10 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) { ; GFX8_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] ; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] -; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX8_DPP-NEXT: v_xor_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX8_DPP-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX8_DPP-NEXT: v_xor_b32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX8_DPP-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX8_DPP-NEXT: s_nop 0 @@ -10085,8 +10007,8 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) { ; GFX8_DPP-NEXT: v_xor_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf ; GFX8_DPP-NEXT: v_readlane_b32 s5, v1, 63 ; GFX8_DPP-NEXT: v_readlane_b32 s4, v2, 63 -; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v2 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v1 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v2 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX8_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5 ; GFX8_DPP-NEXT: ; implicit-def: $vgpr5_vgpr6 @@ -10121,10 +10043,10 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) { ; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] ; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] -; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX9_DPP-NEXT: v_xor_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX9_DPP-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX9_DPP-NEXT: v_xor_b32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX9_DPP-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX9_DPP-NEXT: s_nop 0 @@ -10141,8 +10063,8 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) { ; GFX9_DPP-NEXT: v_xor_b32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf ; GFX9_DPP-NEXT: v_readlane_b32 s5, v1, 63 ; GFX9_DPP-NEXT: v_readlane_b32 s4, v2, 63 -; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v2 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v1 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v2 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5 ; GFX9_DPP-NEXT: ; implicit-def: $vgpr5_vgpr6 @@ -10173,8 +10095,8 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) { ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] ; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX1064_DPP-NEXT: v_xor_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1064_DPP-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1064_DPP-NEXT: v_xor_b32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 @@ -10249,8 +10171,8 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) { ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1 ; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s4 ; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s4 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX1032_DPP-NEXT: v_xor_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1032_DPP-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1032_DPP-NEXT: v_xor_b32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1 @@ -10309,8 +10231,8 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) { ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1] ; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1164_DPP-NEXT: v_xor_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1164_DPP-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 @@ -10398,9 +10320,8 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) { ; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s4 ; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s4 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v5, 0 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, 0 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1132_DPP-NEXT: v_dual_mov_b32 v6, 0 :: v_dual_mov_b32 v5, 0 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1132_DPP-NEXT: v_xor_b32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1132_DPP-NEXT: v_xor_b32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) @@ -11711,71 +11632,66 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) { ; ; GFX8_DPP-LABEL: max_i64_varying: ; GFX8_DPP: ; %bb.0: ; %entry -; GFX8_DPP-NEXT: s_mov_b32 s0, 0 ; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 ; GFX8_DPP-NEXT: v_mov_b32_e32 v9, 0 -; GFX8_DPP-NEXT: s_brev_b32 s1, 1 ; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v7 -; GFX8_DPP-NEXT: s_or_saveexec_b64 s[4:5], -1 -; GFX8_DPP-NEXT: v_mov_b32_e32 v4, s1 -; GFX8_DPP-NEXT: v_bfrev_b32_e32 v1, 1 -; GFX8_DPP-NEXT: v_mov_b32_e32 v3, s0 -; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, v1, 0, s[4:5] -; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5] -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 +; GFX8_DPP-NEXT: v_bfrev_b32_e32 v2, 1 +; GFX8_DPP-NEXT: v_cndmask_b32_e64 v4, v2, 0, s[0:1] +; GFX8_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] +; GFX8_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX8_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX8_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX8_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX8_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v4 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX8_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_readlane_b32 s1, v2, 63 -; GFX8_DPP-NEXT: v_readlane_b32 s0, v1, 63 -; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 wave_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 wave_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: s_mov_b64 exec, s[4:5] -; GFX8_DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v4 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX8_DPP-NEXT: v_readlane_b32 s5, v4, 63 +; GFX8_DPP-NEXT: v_readlane_b32 s4, v3, 63 +; GFX8_DPP-NEXT: v_mov_b32_dpp v2, v4 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v1, v3 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX8_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX8_DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc @@ -11790,9 +11706,9 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) { ; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX8_DPP-NEXT: v_readfirstlane_b32 s5, v8 ; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v7 -; GFX8_DPP-NEXT: v_mov_b32_e32 v7, v3 +; GFX8_DPP-NEXT: v_mov_b32_e32 v7, v1 ; GFX8_DPP-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24 -; GFX8_DPP-NEXT: v_mov_b32_e32 v8, v4 +; GFX8_DPP-NEXT: v_mov_b32_e32 v8, v2 ; GFX8_DPP-NEXT: v_cmp_gt_i64_e32 vcc, s[4:5], v[7:8] ; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s5 ; GFX8_DPP-NEXT: v_cndmask_b32_e32 v8, v8, v0, vcc @@ -11806,71 +11722,66 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) { ; ; GFX9_DPP-LABEL: max_i64_varying: ; GFX9_DPP: ; %bb.0: ; %entry -; GFX9_DPP-NEXT: s_mov_b32 s0, 0 ; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 ; GFX9_DPP-NEXT: v_mov_b32_e32 v9, 0 -; GFX9_DPP-NEXT: s_brev_b32 s1, 1 ; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v7 -; GFX9_DPP-NEXT: s_or_saveexec_b64 s[4:5], -1 -; GFX9_DPP-NEXT: v_mov_b32_e32 v4, s1 -; GFX9_DPP-NEXT: v_bfrev_b32_e32 v1, 1 -; GFX9_DPP-NEXT: v_mov_b32_e32 v3, s0 -; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, v1, 0, s[4:5] -; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5] -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 +; GFX9_DPP-NEXT: v_bfrev_b32_e32 v2, 1 +; GFX9_DPP-NEXT: v_cndmask_b32_e64 v4, v2, 0, s[0:1] +; GFX9_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] +; GFX9_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX9_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX9_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX9_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX9_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v4 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX9_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 0 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_readlane_b32 s1, v2, 63 -; GFX9_DPP-NEXT: v_readlane_b32 s0, v1, 63 -; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 wave_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 wave_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: s_mov_b64 exec, s[4:5] -; GFX9_DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v4 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX9_DPP-NEXT: v_readlane_b32 s5, v4, 63 +; GFX9_DPP-NEXT: v_readlane_b32 s4, v3, 63 +; GFX9_DPP-NEXT: v_mov_b32_dpp v2, v4 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v1, v3 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX9_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX9_DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc @@ -11884,9 +11795,9 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) { ; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX9_DPP-NEXT: v_readfirstlane_b32 s5, v8 ; GFX9_DPP-NEXT: v_readfirstlane_b32 s4, v7 -; GFX9_DPP-NEXT: v_mov_b32_e32 v7, v3 +; GFX9_DPP-NEXT: v_mov_b32_e32 v7, v1 ; GFX9_DPP-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24 -; GFX9_DPP-NEXT: v_mov_b32_e32 v8, v4 +; GFX9_DPP-NEXT: v_mov_b32_e32 v8, v2 ; GFX9_DPP-NEXT: v_cmp_gt_i64_e32 vcc, s[4:5], v[7:8] ; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s5 ; GFX9_DPP-NEXT: v_cndmask_b32_e32 v8, v8, v0, vcc @@ -11901,412 +11812,404 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) { ; GFX1064_DPP-LABEL: max_i64_varying: ; GFX1064_DPP: ; %bb.0: ; %entry ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064_DPP-NEXT: s_mov_b32 s4, 0 -; GFX1064_DPP-NEXT: s_brev_b32 s5, 1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v1, s4 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v2, s5 -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, 0, s[0:1] -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[7:8] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[7:8] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX1064_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1064_DPP-NEXT: v_permlanex16_b32 v8, v3, -1, -1 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v4, 31 -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v3, 31 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, s4 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s5 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, 0, s[0:1] +; GFX1064_DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1] +; GFX1064_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[3:4] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1064_DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[5:6] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GFX1064_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[3:4] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1064_DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[5:6] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GFX1064_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 +; GFX1064_DPP-NEXT: v_permlanex16_b32 v6, v1, -1, -1 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[3:4] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1064_DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v2, 31 +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v1, 31 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, s4 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, s5 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1064_DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1064_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[3:4] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v4, 15 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v3, 15 -; GFX1064_DPP-NEXT: v_readlane_b32 s6, v4, 31 -; GFX1064_DPP-NEXT: v_readlane_b32 s7, v3, 31 -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s4, 16 -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v3, 63 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s5, 16 -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v4, 63 -; GFX1064_DPP-NEXT: v_readlane_b32 s8, v4, 47 -; GFX1064_DPP-NEXT: v_readlane_b32 s9, v3, 47 -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s6, 32 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s7, 32 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v2, 15 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v1, 15 +; GFX1064_DPP-NEXT: v_readlane_b32 s6, v2, 31 +; GFX1064_DPP-NEXT: v_readlane_b32 s7, v1, 31 +; GFX1064_DPP-NEXT: v_writelane_b32 v5, s4, 16 +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v1, 63 +; GFX1064_DPP-NEXT: v_writelane_b32 v4, s5, 16 +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v2, 63 +; GFX1064_DPP-NEXT: v_readlane_b32 s8, v2, 47 +; GFX1064_DPP-NEXT: v_readlane_b32 s9, v1, 47 +; GFX1064_DPP-NEXT: v_writelane_b32 v5, s6, 32 +; GFX1064_DPP-NEXT: v_writelane_b32 v4, s7, 32 ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v9, exec_hi, v0 +; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[6:7], -1 ; GFX1064_DPP-NEXT: s_mov_b64 s[0:1], s[4:5] -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s8, 48 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s9, 48 +; GFX1064_DPP-NEXT: v_writelane_b32 v5, s8, 48 +; GFX1064_DPP-NEXT: v_writelane_b32 v4, s9, 48 ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[6:7] -; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9 +; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX1064_DPP-NEXT: s_mov_b32 s6, -1 -; GFX1064_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10 +; GFX1064_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GFX1064_DPP-NEXT: s_cbranch_execz .LBB23_2 ; GFX1064_DPP-NEXT: ; %bb.1: -; GFX1064_DPP-NEXT: v_mov_b32_e32 v10, s1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, s0 -; GFX1064_DPP-NEXT: ds_max_rtn_i64 v[9:10], v0, v[9:10] +; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, s0 +; GFX1064_DPP-NEXT: ds_max_rtn_i64 v[7:8], v0, v[7:8] ; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064_DPP-NEXT: buffer_gl0_inv ; GFX1064_DPP-NEXT: .LBB23_2: ; GFX1064_DPP-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1064_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x24 -; GFX1064_DPP-NEXT: v_readfirstlane_b32 s1, v10 -; GFX1064_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v10, v2 +; GFX1064_DPP-NEXT: v_readfirstlane_b32 s1, v8 +; GFX1064_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v4 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v5 ; GFX1064_DPP-NEXT: s_mov_b32 s7, 0x31016000 -; GFX1064_DPP-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[9:10] -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v10, v10, s1, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v9, v9, s0, vcc +; GFX1064_DPP-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[7:8] +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v8, v8, s1, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v7, v7, s0, vcc ; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1064_DPP-NEXT: buffer_store_dwordx2 v[9:10], off, s[4:7], 0 +; GFX1064_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[4:7], 0 ; GFX1064_DPP-NEXT: s_endpgm ; ; GFX1032_DPP-LABEL: max_i64_varying: ; GFX1032_DPP: ; %bb.0: ; %entry ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1032_DPP-NEXT: s_mov_b32 s0, 0 -; GFX1032_DPP-NEXT: s_brev_b32 s1, 1 -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, 0, s4 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v2, s1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, s0 -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s4 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[3:4], v[7:8] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[3:4], v[7:8] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc_lo -; GFX1032_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1032_DPP-NEXT: v_permlanex16_b32 v8, v3, -1, -1 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo -; GFX1032_DPP-NEXT: v_readlane_b32 s1, v4, 31 -; GFX1032_DPP-NEXT: v_readlane_b32 s6, v3, 15 -; GFX1032_DPP-NEXT: v_readlane_b32 s0, v3, 31 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_readlane_b32 s5, v4, 15 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, 0, s4 +; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s4 +; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[1:2], v[5:6] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[1:2], v[5:6] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX1032_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 +; GFX1032_DPP-NEXT: v_permlanex16_b32 v6, v1, -1, -1 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1032_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032_DPP-NEXT: v_readlane_b32 s1, v2, 31 +; GFX1032_DPP-NEXT: v_readlane_b32 s6, v1, 15 +; GFX1032_DPP-NEXT: v_readlane_b32 s0, v1, 31 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_readlane_b32 s5, v2, 15 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4 -; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0 +; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 ; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1032_DPP-NEXT: v_writelane_b32 v2, s5, 16 -; GFX1032_DPP-NEXT: v_writelane_b32 v1, s6, 16 +; GFX1032_DPP-NEXT: v_writelane_b32 v5, s5, 16 +; GFX1032_DPP-NEXT: v_writelane_b32 v4, s6, 16 ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4 ; GFX1032_DPP-NEXT: s_mov_b32 s6, -1 -; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v9 -; GFX1032_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10 +; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7 +; GFX1032_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX1032_DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo ; GFX1032_DPP-NEXT: s_cbranch_execz .LBB23_2 ; GFX1032_DPP-NEXT: ; %bb.1: -; GFX1032_DPP-NEXT: v_mov_b32_e32 v10, s1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, s0 -; GFX1032_DPP-NEXT: ds_max_rtn_i64 v[9:10], v0, v[9:10] +; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, s1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, s0 +; GFX1032_DPP-NEXT: ds_max_rtn_i64 v[7:8], v0, v[7:8] ; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032_DPP-NEXT: buffer_gl0_inv ; GFX1032_DPP-NEXT: .LBB23_2: ; GFX1032_DPP-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1032_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x24 -; GFX1032_DPP-NEXT: v_readfirstlane_b32 s1, v10 -; GFX1032_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v10, v2 +; GFX1032_DPP-NEXT: v_readfirstlane_b32 s1, v8 +; GFX1032_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v4 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v5 ; GFX1032_DPP-NEXT: s_mov_b32 s7, 0x31016000 -; GFX1032_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[9:10] -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v10, v10, s1, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v9, v9, s0, vcc_lo +; GFX1032_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[7:8] +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v8, v8, s1, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v7, v7, s0, vcc_lo ; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1032_DPP-NEXT: buffer_store_dwordx2 v[9:10], off, s[4:7], 0 +; GFX1032_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[4:7], 0 ; GFX1032_DPP-NEXT: s_endpgm ; ; GFX1164_DPP-LABEL: max_i64_varying: ; GFX1164_DPP: ; %bb.0: ; %entry -; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164_DPP-NEXT: s_mov_b32 s4, 0 -; GFX1164_DPP-NEXT: s_brev_b32 s5, 1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v1, s4 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, s5 -; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, 0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, 0, s[0:1] +; GFX1164_DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1] +; GFX1164_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[7:8] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v2 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[3:4] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1164_DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[5:6] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GFX1164_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[7:8] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[3:4] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1164_DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1164_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[5:6] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1164_DPP-NEXT: v_permlanex16_b32 v8, v3, -1, -1 +; GFX1164_DPP-NEXT: v_permlanex16_b32 v6, v1, -1, -1 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1164_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[3:4] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1164_DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v2, 31 +; GFX1164_DPP-NEXT: v_readlane_b32 s5, v1, 31 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v4, 31 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v3, 31 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s4 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, s4 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, s5 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, s5 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1164_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[3:4], v[5:6] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1) +; GFX1164_DPP-NEXT: v_cmp_gt_i64_e32 vcc, v[1:2], v[3:4] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v4, 15 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v3, 15 -; GFX1164_DPP-NEXT: v_readlane_b32 s6, v4, 31 -; GFX1164_DPP-NEXT: v_readlane_b32 s7, v3, 31 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s4, 16 -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v3, 63 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s5, 16 -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v4, 63 -; GFX1164_DPP-NEXT: v_readlane_b32 s8, v4, 47 -; GFX1164_DPP-NEXT: v_readlane_b32 s9, v3, 47 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s6, 32 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s7, 32 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v2, 15 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_readlane_b32 s5, v1, 15 +; GFX1164_DPP-NEXT: v_readlane_b32 s6, v2, 31 +; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 31 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s4, 16 +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v1, 63 +; GFX1164_DPP-NEXT: v_writelane_b32 v4, s5, 16 +; GFX1164_DPP-NEXT: v_readlane_b32 s5, v2, 63 +; GFX1164_DPP-NEXT: v_readlane_b32 s8, v2, 47 +; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 47 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32 +; GFX1164_DPP-NEXT: v_writelane_b32 v4, s7, 32 ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v9, exec_hi, v0 +; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[6:7], -1 ; GFX1164_DPP-NEXT: s_mov_b64 s[0:1], s[4:5] -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s8, 48 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s9, 48 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s8, 48 +; GFX1164_DPP-NEXT: v_writelane_b32 v4, s9, 48 ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7] -; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9 +; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX1164_DPP-NEXT: s_mov_b32 s6, -1 -; GFX1164_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10 +; GFX1164_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GFX1164_DPP-NEXT: s_cbranch_execz .LBB23_2 ; GFX1164_DPP-NEXT: ; %bb.1: -; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, s1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, s0 -; GFX1164_DPP-NEXT: ds_max_rtn_i64 v[9:10], v0, v[9:10] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, s1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s0 +; GFX1164_DPP-NEXT: ds_max_rtn_i64 v[7:8], v0, v[7:8] ; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164_DPP-NEXT: buffer_gl0_inv ; GFX1164_DPP-NEXT: .LBB23_2: ; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1164_DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x24 -; GFX1164_DPP-NEXT: v_readfirstlane_b32 s1, v10 -; GFX1164_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, v2 +; GFX1164_DPP-NEXT: v_readfirstlane_b32 s1, v8 +; GFX1164_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, v4 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v5 ; GFX1164_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[9:10] -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v10, v10, s1, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v9, v9, s0, vcc +; GFX1164_DPP-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[7:8] +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v8, v8, s1, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v7, v7, s0, vcc ; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1164_DPP-NEXT: buffer_store_b64 v[9:10], off, s[4:7], 0 +; GFX1164_DPP-NEXT: buffer_store_b64 v[7:8], off, s[4:7], 0 ; GFX1164_DPP-NEXT: s_endpgm ; ; GFX1132_DPP-LABEL: max_i64_varying: ; GFX1132_DPP: ; %bb.0: ; %entry -; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1132_DPP-NEXT: s_mov_b32 s0, 0 -; GFX1132_DPP-NEXT: s_brev_b32 s1, 1 -; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1) -; GFX1132_DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 -; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4 ; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, 0, s4 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v6, v2 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s4 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v8, v2 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v3, v5, v3 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v6, v2 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[3:4], v[7:8] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v8, v4 :: v_dual_cndmask_b32 v3, v7, v3 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v8, v2 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v3, v5, v3 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v6, v2 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[3:4], v[7:8] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v8, v4 :: v_dual_cndmask_b32 v3, v7, v3 -; GFX1132_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_permlanex16_b32 v8, v3, -1, -1 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1132_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v3, v5, v3 +; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, 0, s4 +; GFX1132_DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s4 +; GFX1132_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX1132_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v2, v4, v2 :: v_dual_cndmask_b32 v1, v3, v1 +; GFX1132_DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[1:2], v[5:6] +; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v2, v6, v2 :: v_dual_cndmask_b32 v1, v5, v1 +; GFX1132_DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX1132_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v2, v4, v2 :: v_dual_cndmask_b32 v1, v3, v1 +; GFX1132_DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[1:2], v[5:6] +; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v2, v6, v2 :: v_dual_cndmask_b32 v1, v5, v1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_readlane_b32 s1, v4, 31 -; GFX1132_DPP-NEXT: v_readlane_b32 s6, v3, 15 -; GFX1132_DPP-NEXT: v_readlane_b32 s0, v3, 31 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_readlane_b32 s5, v4, 15 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 +; GFX1132_DPP-NEXT: v_permlanex16_b32 v6, v1, -1, -1 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1132_DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1132_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo +; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_cndmask_b32 v1, v3, v1 +; GFX1132_DPP-NEXT: v_readlane_b32 s1, v2, 31 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(SALU_CYCLE_1) +; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15 +; GFX1132_DPP-NEXT: v_readlane_b32 s0, v1, 31 +; GFX1132_DPP-NEXT: v_readlane_b32 s5, v2, 15 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4 -; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0 +; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 ; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1132_DPP-NEXT: v_writelane_b32 v2, s5, 16 -; GFX1132_DPP-NEXT: v_writelane_b32 v1, s6, 16 +; GFX1132_DPP-NEXT: v_writelane_b32 v5, s5, 16 +; GFX1132_DPP-NEXT: v_writelane_b32 v4, s6, 16 ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4 ; GFX1132_DPP-NEXT: s_mov_b32 s6, -1 -; GFX1132_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v9 -; GFX1132_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10 +; GFX1132_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7 +; GFX1132_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX1132_DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo ; GFX1132_DPP-NEXT: s_cbranch_execz .LBB23_2 ; GFX1132_DPP-NEXT: ; %bb.1: -; GFX1132_DPP-NEXT: v_dual_mov_b32 v10, s1 :: v_dual_mov_b32 v9, s0 -; GFX1132_DPP-NEXT: ds_max_rtn_i64 v[9:10], v0, v[9:10] +; GFX1132_DPP-NEXT: v_dual_mov_b32 v8, s1 :: v_dual_mov_b32 v7, s0 +; GFX1132_DPP-NEXT: ds_max_rtn_i64 v[7:8], v0, v[7:8] ; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132_DPP-NEXT: buffer_gl0_inv ; GFX1132_DPP-NEXT: .LBB23_2: ; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1132_DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x24 -; GFX1132_DPP-NEXT: v_readfirstlane_b32 s1, v10 -; GFX1132_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v9, v1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v10, v2 +; GFX1132_DPP-NEXT: v_readfirstlane_b32 s1, v8 +; GFX1132_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v7, v4 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v8, v5 ; GFX1132_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[9:10] -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v10, v10, s1, vcc_lo -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v9, v9, s0, vcc_lo +; GFX1132_DPP-NEXT: v_cmp_gt_i64_e32 vcc_lo, s[0:1], v[7:8] +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v8, v8, s1, vcc_lo +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v7, v7, s0, vcc_lo ; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1132_DPP-NEXT: buffer_store_b64 v[9:10], off, s[4:7], 0 +; GFX1132_DPP-NEXT: buffer_store_b64 v[7:8], off, s[4:7], 0 ; GFX1132_DPP-NEXT: s_endpgm entry: %lane = call i32 @llvm.amdgcn.workitem.id.x() @@ -13566,65 +13469,61 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) { ; GFX8_DPP-NEXT: v_mov_b32_e32 v9, 0 ; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v7 ; GFX8_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX8_DPP-NEXT: s_mov_b32 s6, -1 -; GFX8_DPP-NEXT: s_brev_b32 s7, -2 -; GFX8_DPP-NEXT: v_mov_b32_e32 v3, s6 -; GFX8_DPP-NEXT: v_bfrev_b32_e32 v1, -2 -; GFX8_DPP-NEXT: v_mov_b32_e32 v4, s7 -; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, v1, 0, s[0:1] -; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s[0:1] -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 -; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_bfrev_b32_e32 v2, -2 +; GFX8_DPP-NEXT: v_cndmask_b32_e64 v4, v2, 0, s[0:1] +; GFX8_DPP-NEXT: v_cndmask_b32_e64 v3, -1, v0, s[0:1] +; GFX8_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX8_DPP-NEXT: v_mov_b32_e32 v1, -1 +; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX8_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, -1 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX8_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, -1 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX8_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, -1 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX8_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, -1 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v4 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX8_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, -1 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_readlane_b32 s5, v2, 63 -; GFX8_DPP-NEXT: v_readlane_b32 s4, v1, 63 -; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 wave_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v4 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX8_DPP-NEXT: v_readlane_b32 s5, v4, 63 +; GFX8_DPP-NEXT: v_readlane_b32 s4, v3, 63 +; GFX8_DPP-NEXT: v_mov_b32_dpp v2, v4 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v1, v3 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX8_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX8_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 @@ -13638,19 +13537,20 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) { ; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX8_DPP-NEXT: .LBB26_2: ; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[0:1] -; GFX8_DPP-NEXT: v_readfirstlane_b32 s1, v8 -; GFX8_DPP-NEXT: v_readfirstlane_b32 s0, v7 -; GFX8_DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX8_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x24 -; GFX8_DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX8_DPP-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[7:8] -; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s1 +; GFX8_DPP-NEXT: v_readfirstlane_b32 s5, v8 +; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v7 +; GFX8_DPP-NEXT: v_mov_b32_e32 v7, v1 +; GFX8_DPP-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24 +; GFX8_DPP-NEXT: v_mov_b32_e32 v8, v2 +; GFX8_DPP-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[7:8] +; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s5 ; GFX8_DPP-NEXT: v_cndmask_b32_e32 v8, v8, v0, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s0 -; GFX8_DPP-NEXT: s_mov_b32 s7, 0xf000 +; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s4 +; GFX8_DPP-NEXT: s_mov_b32 s3, 0xf000 +; GFX8_DPP-NEXT: s_mov_b32 s2, -1 ; GFX8_DPP-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc ; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX8_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[4:7], 0 +; GFX8_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[0:3], 0 ; GFX8_DPP-NEXT: s_endpgm ; ; GFX9_DPP-LABEL: min_i64_varying: @@ -13659,65 +13559,61 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) { ; GFX9_DPP-NEXT: v_mov_b32_e32 v9, 0 ; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v7 ; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9_DPP-NEXT: s_mov_b32 s6, -1 -; GFX9_DPP-NEXT: s_brev_b32 s7, -2 -; GFX9_DPP-NEXT: v_mov_b32_e32 v3, s6 -; GFX9_DPP-NEXT: v_bfrev_b32_e32 v1, -2 -; GFX9_DPP-NEXT: v_mov_b32_e32 v4, s7 -; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, v1, 0, s[0:1] -; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s[0:1] -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 -; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_bfrev_b32_e32 v2, -2 +; GFX9_DPP-NEXT: v_cndmask_b32_e64 v4, v2, 0, s[0:1] +; GFX9_DPP-NEXT: v_cndmask_b32_e64 v3, -1, v0, s[0:1] +; GFX9_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX9_DPP-NEXT: v_mov_b32_e32 v1, -1 +; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX9_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, -1 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX9_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, -1 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX9_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, -1 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX9_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, -1 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v4 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX9_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, -1 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_readlane_b32 s5, v2, 63 -; GFX9_DPP-NEXT: v_readlane_b32 s4, v1, 63 -; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 wave_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v4 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc +; GFX9_DPP-NEXT: v_readlane_b32 s5, v4, 63 +; GFX9_DPP-NEXT: v_readlane_b32 s4, v3, 63 +; GFX9_DPP-NEXT: v_mov_b32_dpp v2, v4 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v1, v3 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX9_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 @@ -13730,426 +13626,423 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) { ; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9_DPP-NEXT: .LBB26_2: ; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[0:1] -; GFX9_DPP-NEXT: v_readfirstlane_b32 s1, v8 -; GFX9_DPP-NEXT: v_readfirstlane_b32 s0, v7 -; GFX9_DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX9_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x24 -; GFX9_DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX9_DPP-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[7:8] -; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s1 +; GFX9_DPP-NEXT: v_readfirstlane_b32 s5, v8 +; GFX9_DPP-NEXT: v_readfirstlane_b32 s4, v7 +; GFX9_DPP-NEXT: v_mov_b32_e32 v7, v1 +; GFX9_DPP-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24 +; GFX9_DPP-NEXT: v_mov_b32_e32 v8, v2 +; GFX9_DPP-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[7:8] +; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s5 ; GFX9_DPP-NEXT: v_cndmask_b32_e32 v8, v8, v0, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s0 -; GFX9_DPP-NEXT: s_mov_b32 s7, 0xf000 +; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s4 +; GFX9_DPP-NEXT: s_mov_b32 s3, 0xf000 +; GFX9_DPP-NEXT: s_mov_b32 s2, -1 ; GFX9_DPP-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc ; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX9_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[4:7], 0 +; GFX9_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[0:3], 0 ; GFX9_DPP-NEXT: s_endpgm ; ; GFX1064_DPP-LABEL: min_i64_varying: ; GFX1064_DPP: ; %bb.0: ; %entry ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064_DPP-NEXT: s_mov_b32 s6, -1 -; GFX1064_DPP-NEXT: s_brev_b32 s7, -2 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v1, s6 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v2, s7 -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fffffff, 0, s[0:1] -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v3, -1, v0, s[0:1] -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[7:8] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[7:8] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX1064_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1064_DPP-NEXT: v_permlanex16_b32 v8, v3, -1, -1 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v4, 31 -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v3, 31 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, s4 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s5 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fffffff, 0, s[0:1] +; GFX1064_DPP-NEXT: v_bfrev_b32_e32 v4, -2 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s[0:1] +; GFX1064_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[3:4] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1064_DPP-NEXT: v_bfrev_b32_e32 v4, -2 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[5:6] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GFX1064_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[3:4] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1064_DPP-NEXT: v_bfrev_b32_e32 v4, -2 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[5:6] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GFX1064_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 +; GFX1064_DPP-NEXT: v_permlanex16_b32 v6, v1, -1, -1 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[3:4] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1064_DPP-NEXT: v_bfrev_b32_e32 v4, -2 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v2, 31 +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v1, 31 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, s4 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, s5 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1064_DPP-NEXT: v_bfrev_b32_e32 v5, -2 +; GFX1064_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[3:4] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v4, 15 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v3, 15 -; GFX1064_DPP-NEXT: v_readlane_b32 s7, v4, 31 -; GFX1064_DPP-NEXT: v_readlane_b32 s8, v3, 31 -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s4, 16 -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v3, 63 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s5, 16 -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v4, 63 -; GFX1064_DPP-NEXT: v_readlane_b32 s10, v4, 47 -; GFX1064_DPP-NEXT: v_readlane_b32 s11, v3, 47 -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s7, 32 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s8, 32 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v2, 15 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v1, 15 +; GFX1064_DPP-NEXT: v_readlane_b32 s6, v2, 31 +; GFX1064_DPP-NEXT: v_readlane_b32 s7, v1, 31 +; GFX1064_DPP-NEXT: v_writelane_b32 v5, s4, 16 +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v1, 63 +; GFX1064_DPP-NEXT: v_writelane_b32 v4, s5, 16 +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v2, 63 +; GFX1064_DPP-NEXT: v_readlane_b32 s8, v2, 47 +; GFX1064_DPP-NEXT: v_readlane_b32 s9, v1, 47 +; GFX1064_DPP-NEXT: v_writelane_b32 v5, s6, 32 +; GFX1064_DPP-NEXT: v_writelane_b32 v4, s7, 32 ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v9, exec_hi, v0 +; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, 0 -; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[8:9], -1 +; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[6:7], -1 ; GFX1064_DPP-NEXT: s_mov_b64 s[0:1], s[4:5] -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s10, 48 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s11, 48 -; GFX1064_DPP-NEXT: s_mov_b64 exec, s[8:9] -; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9 -; GFX1064_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10 +; GFX1064_DPP-NEXT: v_writelane_b32 v5, s8, 48 +; GFX1064_DPP-NEXT: v_writelane_b32 v4, s9, 48 +; GFX1064_DPP-NEXT: s_mov_b64 exec, s[6:7] +; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 +; GFX1064_DPP-NEXT: s_mov_b32 s6, -1 +; GFX1064_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GFX1064_DPP-NEXT: s_cbranch_execz .LBB26_2 ; GFX1064_DPP-NEXT: ; %bb.1: -; GFX1064_DPP-NEXT: v_mov_b32_e32 v10, s1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, s0 -; GFX1064_DPP-NEXT: ds_min_rtn_i64 v[9:10], v0, v[9:10] +; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, s0 +; GFX1064_DPP-NEXT: ds_min_rtn_i64 v[7:8], v0, v[7:8] ; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064_DPP-NEXT: buffer_gl0_inv ; GFX1064_DPP-NEXT: .LBB26_2: ; GFX1064_DPP-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1064_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x24 -; GFX1064_DPP-NEXT: v_readfirstlane_b32 s1, v10 -; GFX1064_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v10, v2 +; GFX1064_DPP-NEXT: v_readfirstlane_b32 s1, v8 +; GFX1064_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v4 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v5 ; GFX1064_DPP-NEXT: s_mov_b32 s7, 0x31016000 -; GFX1064_DPP-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[9:10] -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v10, v10, s1, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v9, v9, s0, vcc +; GFX1064_DPP-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[7:8] +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v8, v8, s1, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v7, v7, s0, vcc ; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1064_DPP-NEXT: buffer_store_dwordx2 v[9:10], off, s[4:7], 0 +; GFX1064_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[4:7], 0 ; GFX1064_DPP-NEXT: s_endpgm ; ; GFX1032_DPP-LABEL: min_i64_varying: ; GFX1032_DPP: ; %bb.0: ; %entry ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1032_DPP-NEXT: s_mov_b32 s6, -1 -; GFX1032_DPP-NEXT: s_brev_b32 s7, -2 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, s6 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v2, s7 -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fffffff, 0, s4 -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v3, -1, v0, s4 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[3:4], v[7:8] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[3:4], v[7:8] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc_lo -; GFX1032_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1032_DPP-NEXT: v_permlanex16_b32 v8, v3, -1, -1 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo -; GFX1032_DPP-NEXT: v_readlane_b32 s1, v4, 31 -; GFX1032_DPP-NEXT: v_readlane_b32 s0, v3, 31 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_readlane_b32 s5, v4, 15 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_readlane_b32 s7, v3, 15 +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fffffff, 0, s4 +; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v4, -2 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s4 +; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v4, -2 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[1:2], v[5:6] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v4, -2 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[1:2], v[5:6] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX1032_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 +; GFX1032_DPP-NEXT: v_permlanex16_b32 v6, v1, -1, -1 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v5, -2 +; GFX1032_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX1032_DPP-NEXT: v_readlane_b32 s1, v2, 31 +; GFX1032_DPP-NEXT: v_readlane_b32 s6, v1, 15 +; GFX1032_DPP-NEXT: v_readlane_b32 s0, v1, 31 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_readlane_b32 s5, v2, 15 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4 -; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0 +; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 ; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1032_DPP-NEXT: v_writelane_b32 v2, s5, 16 -; GFX1032_DPP-NEXT: v_writelane_b32 v1, s7, 16 +; GFX1032_DPP-NEXT: v_writelane_b32 v5, s5, 16 +; GFX1032_DPP-NEXT: v_writelane_b32 v4, s6, 16 ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4 -; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v9 -; GFX1032_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10 +; GFX1032_DPP-NEXT: s_mov_b32 s6, -1 +; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7 +; GFX1032_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX1032_DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo ; GFX1032_DPP-NEXT: s_cbranch_execz .LBB26_2 ; GFX1032_DPP-NEXT: ; %bb.1: -; GFX1032_DPP-NEXT: v_mov_b32_e32 v10, s1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, s0 -; GFX1032_DPP-NEXT: ds_min_rtn_i64 v[9:10], v0, v[9:10] +; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, s1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, s0 +; GFX1032_DPP-NEXT: ds_min_rtn_i64 v[7:8], v0, v[7:8] ; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032_DPP-NEXT: buffer_gl0_inv ; GFX1032_DPP-NEXT: .LBB26_2: ; GFX1032_DPP-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1032_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x24 -; GFX1032_DPP-NEXT: v_readfirstlane_b32 s1, v10 -; GFX1032_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v10, v2 +; GFX1032_DPP-NEXT: v_readfirstlane_b32 s1, v8 +; GFX1032_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v4 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v5 ; GFX1032_DPP-NEXT: s_mov_b32 s7, 0x31016000 -; GFX1032_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[0:1], v[9:10] -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v10, v10, s1, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v9, v9, s0, vcc_lo +; GFX1032_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[0:1], v[7:8] +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v8, v8, s1, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v7, v7, s0, vcc_lo ; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1032_DPP-NEXT: buffer_store_dwordx2 v[9:10], off, s[4:7], 0 +; GFX1032_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[4:7], 0 ; GFX1032_DPP-NEXT: s_endpgm ; ; GFX1164_DPP-LABEL: min_i64_varying: ; GFX1164_DPP: ; %bb.0: ; %entry -; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164_DPP-NEXT: s_mov_b32 s6, -1 -; GFX1164_DPP-NEXT: s_brev_b32 s7, -2 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v1, s6 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, s7 -; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fffffff, 0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v3, -1, v0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fffffff, 0, s[0:1] +; GFX1164_DPP-NEXT: v_bfrev_b32_e32 v4, -2 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s[0:1] +; GFX1164_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[7:8] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v2 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[3:4] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1164_DPP-NEXT: v_bfrev_b32_e32 v4, -2 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[5:6] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GFX1164_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[7:8] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[3:4] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1164_DPP-NEXT: v_bfrev_b32_e32 v4, -2 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1164_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[5:6] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1164_DPP-NEXT: v_permlanex16_b32 v8, v3, -1, -1 +; GFX1164_DPP-NEXT: v_permlanex16_b32 v6, v1, -1, -1 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1164_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[3:4] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1164_DPP-NEXT: v_bfrev_b32_e32 v4, -2 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v2, 31 +; GFX1164_DPP-NEXT: v_readlane_b32 s5, v1, 31 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v4, 31 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v3, 31 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s4 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, s4 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, s5 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, s5 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1164_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[3:4], v[5:6] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: v_bfrev_b32_e32 v5, -2 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1) +; GFX1164_DPP-NEXT: v_cmp_lt_i64_e32 vcc, v[1:2], v[3:4] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v4, 15 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v3, 15 -; GFX1164_DPP-NEXT: v_readlane_b32 s7, v4, 31 -; GFX1164_DPP-NEXT: v_readlane_b32 s8, v3, 31 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s4, 16 -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v3, 63 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s5, 16 -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v4, 63 -; GFX1164_DPP-NEXT: v_readlane_b32 s10, v4, 47 -; GFX1164_DPP-NEXT: v_readlane_b32 s11, v3, 47 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s7, 32 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s8, 32 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v2, 15 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_readlane_b32 s5, v1, 15 +; GFX1164_DPP-NEXT: v_readlane_b32 s6, v2, 31 +; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 31 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s4, 16 +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v1, 63 +; GFX1164_DPP-NEXT: v_writelane_b32 v4, s5, 16 +; GFX1164_DPP-NEXT: v_readlane_b32 s5, v2, 63 +; GFX1164_DPP-NEXT: v_readlane_b32 s8, v2, 47 +; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 47 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32 +; GFX1164_DPP-NEXT: v_writelane_b32 v4, s7, 32 ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v9, exec_hi, v0 +; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0 -; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[8:9], -1 +; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[6:7], -1 ; GFX1164_DPP-NEXT: s_mov_b64 s[0:1], s[4:5] -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s10, 48 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s11, 48 -; GFX1164_DPP-NEXT: s_mov_b64 exec, s[8:9] -; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9 -; GFX1164_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s8, 48 +; GFX1164_DPP-NEXT: v_writelane_b32 v4, s9, 48 +; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7] +; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 +; GFX1164_DPP-NEXT: s_mov_b32 s6, -1 +; GFX1164_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GFX1164_DPP-NEXT: s_cbranch_execz .LBB26_2 ; GFX1164_DPP-NEXT: ; %bb.1: -; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, s1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, s0 -; GFX1164_DPP-NEXT: ds_min_rtn_i64 v[9:10], v0, v[9:10] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, s1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s0 +; GFX1164_DPP-NEXT: ds_min_rtn_i64 v[7:8], v0, v[7:8] ; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164_DPP-NEXT: buffer_gl0_inv ; GFX1164_DPP-NEXT: .LBB26_2: ; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1164_DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x24 -; GFX1164_DPP-NEXT: v_readfirstlane_b32 s1, v10 -; GFX1164_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, v2 +; GFX1164_DPP-NEXT: v_readfirstlane_b32 s1, v8 +; GFX1164_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, v4 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v5 ; GFX1164_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[9:10] -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v10, v10, s1, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v9, v9, s0, vcc +; GFX1164_DPP-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[7:8] +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v8, v8, s1, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v7, v7, s0, vcc ; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1164_DPP-NEXT: buffer_store_b64 v[9:10], off, s[4:7], 0 +; GFX1164_DPP-NEXT: buffer_store_b64 v[7:8], off, s[4:7], 0 ; GFX1164_DPP-NEXT: s_endpgm ; ; GFX1132_DPP-LABEL: min_i64_varying: ; GFX1132_DPP: ; %bb.0: ; %entry -; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132_DPP-NEXT: s_brev_b32 s7, -2 -; GFX1132_DPP-NEXT: s_mov_b32 s6, -1 -; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1) -; GFX1132_DPP-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v2, s7 -; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fffffff, 0, s4 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v6, v2 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v3, -1, v0, s4 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v8, v2 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v3, v5, v3 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v6, v2 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[3:4], v[7:8] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v8, v4 :: v_dual_cndmask_b32 v3, v7, v3 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v8, v2 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v3, v5, v3 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v6, v2 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[3:4], v[7:8] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v8, v4 :: v_dual_cndmask_b32 v3, v7, v3 -; GFX1132_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_permlanex16_b32 v8, v3, -1, -1 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1132_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v3, v5, v3 +; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fffffff, 0, s4 +; GFX1132_DPP-NEXT: v_bfrev_b32_e32 v4, -2 +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s4 +; GFX1132_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX1132_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v2, v4, v2 :: v_dual_cndmask_b32 v1, v3, v1 +; GFX1132_DPP-NEXT: v_bfrev_b32_e32 v4, -2 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[1:2], v[5:6] +; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v2, v6, v2 :: v_dual_cndmask_b32 v1, v5, v1 +; GFX1132_DPP-NEXT: v_bfrev_b32_e32 v6, -2 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX1132_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v2, v4, v2 :: v_dual_cndmask_b32 v1, v3, v1 +; GFX1132_DPP-NEXT: v_bfrev_b32_e32 v4, -2 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[1:2], v[5:6] +; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v2, v6, v2 :: v_dual_cndmask_b32 v1, v5, v1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_readlane_b32 s1, v4, 31 -; GFX1132_DPP-NEXT: v_readlane_b32 s0, v3, 31 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_readlane_b32 s5, v4, 15 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_readlane_b32 s7, v3, 15 +; GFX1132_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 +; GFX1132_DPP-NEXT: v_permlanex16_b32 v6, v1, -1, -1 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1132_DPP-NEXT: v_bfrev_b32_e32 v5, -2 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1132_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo +; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, -1 :: v_dual_cndmask_b32 v1, v3, v1 +; GFX1132_DPP-NEXT: v_readlane_b32 s1, v2, 31 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(SALU_CYCLE_1) +; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15 +; GFX1132_DPP-NEXT: v_readlane_b32 s0, v1, 31 +; GFX1132_DPP-NEXT: v_readlane_b32 s5, v2, 15 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4 -; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0 +; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 ; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1132_DPP-NEXT: v_writelane_b32 v2, s5, 16 -; GFX1132_DPP-NEXT: v_writelane_b32 v1, s7, 16 +; GFX1132_DPP-NEXT: v_writelane_b32 v5, s5, 16 +; GFX1132_DPP-NEXT: v_writelane_b32 v4, s6, 16 ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4 -; GFX1132_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v9 -; GFX1132_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10 +; GFX1132_DPP-NEXT: s_mov_b32 s6, -1 +; GFX1132_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7 +; GFX1132_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX1132_DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo ; GFX1132_DPP-NEXT: s_cbranch_execz .LBB26_2 ; GFX1132_DPP-NEXT: ; %bb.1: -; GFX1132_DPP-NEXT: v_dual_mov_b32 v10, s1 :: v_dual_mov_b32 v9, s0 -; GFX1132_DPP-NEXT: ds_min_rtn_i64 v[9:10], v0, v[9:10] +; GFX1132_DPP-NEXT: v_dual_mov_b32 v8, s1 :: v_dual_mov_b32 v7, s0 +; GFX1132_DPP-NEXT: ds_min_rtn_i64 v[7:8], v0, v[7:8] ; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132_DPP-NEXT: buffer_gl0_inv ; GFX1132_DPP-NEXT: .LBB26_2: ; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1132_DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x24 -; GFX1132_DPP-NEXT: v_readfirstlane_b32 s1, v10 -; GFX1132_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v9, v1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v10, v2 +; GFX1132_DPP-NEXT: v_readfirstlane_b32 s1, v8 +; GFX1132_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v7, v4 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v8, v5 ; GFX1132_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[0:1], v[9:10] -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v10, v10, s1, vcc_lo -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v9, v9, s0, vcc_lo +; GFX1132_DPP-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[0:1], v[7:8] +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v8, v8, s1, vcc_lo +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v7, v7, s0, vcc_lo ; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1132_DPP-NEXT: buffer_store_b64 v[9:10], off, s[4:7], 0 +; GFX1132_DPP-NEXT: buffer_store_b64 v[7:8], off, s[4:7], 0 ; GFX1132_DPP-NEXT: s_endpgm entry: %lane = call i32 @llvm.amdgcn.workitem.id.x() @@ -15395,587 +15288,581 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) { ; ; GFX8_DPP-LABEL: umax_i64_varying: ; GFX8_DPP: ; %bb.0: ; %entry -; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 -; GFX8_DPP-NEXT: v_mov_b32_e32 v9, 0 -; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v7 +; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v5, exec_lo, 0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v5, exec_hi, v5 ; GFX8_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 -; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[0:1] ; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1] -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX8_DPP-NEXT: v_readlane_b32 s5, v2, 63 ; GFX8_DPP-NEXT: v_readlane_b32 s4, v1, 63 -; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX8_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 -; GFX8_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 +; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5 +; GFX8_DPP-NEXT: ; implicit-def: $vgpr5_vgpr6 ; GFX8_DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX8_DPP-NEXT: s_cbranch_execz .LBB29_2 ; GFX8_DPP-NEXT: ; %bb.1: -; GFX8_DPP-NEXT: v_mov_b32_e32 v8, s5 -; GFX8_DPP-NEXT: v_mov_b32_e32 v7, s4 +; GFX8_DPP-NEXT: v_mov_b32_e32 v6, s5 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, s4 ; GFX8_DPP-NEXT: s_mov_b32 m0, -1 -; GFX8_DPP-NEXT: ds_max_rtn_u64 v[7:8], v9, v[7:8] +; GFX8_DPP-NEXT: ds_max_rtn_u64 v[5:6], v7, v[5:6] ; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX8_DPP-NEXT: .LBB29_2: ; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[0:1] -; GFX8_DPP-NEXT: v_readfirstlane_b32 s5, v8 -; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v7 -; GFX8_DPP-NEXT: v_mov_b32_e32 v7, v3 +; GFX8_DPP-NEXT: v_readfirstlane_b32 s5, v6 +; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v5 +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 ; GFX8_DPP-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24 -; GFX8_DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX8_DPP-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[7:8] +; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[5:6] ; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s5 -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v8, v8, v0, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc ; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s4 ; GFX8_DPP-NEXT: s_mov_b32 s3, 0xf000 ; GFX8_DPP-NEXT: s_mov_b32 s2, -1 -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX8_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[0:3], 0 +; GFX8_DPP-NEXT: buffer_store_dwordx2 v[5:6], off, s[0:3], 0 ; GFX8_DPP-NEXT: s_endpgm ; ; GFX9_DPP-LABEL: umax_i64_varying: ; GFX9_DPP: ; %bb.0: ; %entry -; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 -; GFX9_DPP-NEXT: v_mov_b32_e32 v9, 0 -; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v7 +; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v5, exec_lo, 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v5, exec_hi, v5 ; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 -; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[0:1] ; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1] -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX9_DPP-NEXT: v_readlane_b32 s5, v2, 63 ; GFX9_DPP-NEXT: v_readlane_b32 s4, v1, 63 -; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 -; GFX9_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 +; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5 +; GFX9_DPP-NEXT: ; implicit-def: $vgpr5_vgpr6 ; GFX9_DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX9_DPP-NEXT: s_cbranch_execz .LBB29_2 ; GFX9_DPP-NEXT: ; %bb.1: -; GFX9_DPP-NEXT: v_mov_b32_e32 v8, s5 -; GFX9_DPP-NEXT: v_mov_b32_e32 v7, s4 -; GFX9_DPP-NEXT: ds_max_rtn_u64 v[7:8], v9, v[7:8] +; GFX9_DPP-NEXT: v_mov_b32_e32 v6, s5 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, s4 +; GFX9_DPP-NEXT: ds_max_rtn_u64 v[5:6], v7, v[5:6] ; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9_DPP-NEXT: .LBB29_2: ; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[0:1] -; GFX9_DPP-NEXT: v_readfirstlane_b32 s5, v8 -; GFX9_DPP-NEXT: v_readfirstlane_b32 s4, v7 -; GFX9_DPP-NEXT: v_mov_b32_e32 v7, v3 +; GFX9_DPP-NEXT: v_readfirstlane_b32 s5, v6 +; GFX9_DPP-NEXT: v_readfirstlane_b32 s4, v5 +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 ; GFX9_DPP-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24 -; GFX9_DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX9_DPP-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[7:8] +; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[5:6] ; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s5 -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v8, v8, v0, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc ; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s4 ; GFX9_DPP-NEXT: s_mov_b32 s3, 0xf000 ; GFX9_DPP-NEXT: s_mov_b32 s2, -1 -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc ; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX9_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[0:3], 0 +; GFX9_DPP-NEXT: buffer_store_dwordx2 v[5:6], off, s[0:3], 0 ; GFX9_DPP-NEXT: s_endpgm ; ; GFX1064_DPP-LABEL: umax_i64_varying: ; GFX1064_DPP: ; %bb.0: ; %entry ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v4, 0, 0, s[0:1] -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[3:4], v[5:6] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[3:4], v[7:8] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[3:4], v[5:6] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[3:4], v[7:8] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX1064_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1064_DPP-NEXT: v_permlanex16_b32 v8, v3, -1, -1 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[3:4], v[5:6] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v4, 31 -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v3, 31 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, s4 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s5 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[3:4], v[5:6] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[0:1] +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1] +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[5:6] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[5:6] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GFX1064_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 +; GFX1064_DPP-NEXT: v_permlanex16_b32 v6, v1, -1, -1 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v2, 31 +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v1, 31 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, s4 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, s5 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v4, 15 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v3, 15 -; GFX1064_DPP-NEXT: v_readlane_b32 s6, v4, 31 -; GFX1064_DPP-NEXT: v_readlane_b32 s7, v3, 31 -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s4, 16 -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v3, 63 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s5, 16 -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v4, 63 -; GFX1064_DPP-NEXT: v_readlane_b32 s8, v4, 47 -; GFX1064_DPP-NEXT: v_readlane_b32 s9, v3, 47 -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s6, 32 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s7, 32 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v2, 15 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v1, 15 +; GFX1064_DPP-NEXT: v_readlane_b32 s6, v2, 31 +; GFX1064_DPP-NEXT: v_readlane_b32 s7, v1, 31 +; GFX1064_DPP-NEXT: v_writelane_b32 v5, s4, 16 +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v1, 63 +; GFX1064_DPP-NEXT: v_writelane_b32 v4, s5, 16 +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v2, 63 +; GFX1064_DPP-NEXT: v_readlane_b32 s8, v2, 47 +; GFX1064_DPP-NEXT: v_readlane_b32 s9, v1, 47 +; GFX1064_DPP-NEXT: v_writelane_b32 v5, s6, 32 +; GFX1064_DPP-NEXT: v_writelane_b32 v4, s7, 32 ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v9, exec_hi, v0 +; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[6:7], -1 ; GFX1064_DPP-NEXT: s_mov_b64 s[0:1], s[4:5] -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s8, 48 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s9, 48 +; GFX1064_DPP-NEXT: v_writelane_b32 v5, s8, 48 +; GFX1064_DPP-NEXT: v_writelane_b32 v4, s9, 48 ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[6:7] -; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9 +; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX1064_DPP-NEXT: s_mov_b32 s6, -1 -; GFX1064_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10 +; GFX1064_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GFX1064_DPP-NEXT: s_cbranch_execz .LBB29_2 ; GFX1064_DPP-NEXT: ; %bb.1: -; GFX1064_DPP-NEXT: v_mov_b32_e32 v10, s1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, s0 -; GFX1064_DPP-NEXT: ds_max_rtn_u64 v[9:10], v0, v[9:10] +; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, s0 +; GFX1064_DPP-NEXT: ds_max_rtn_u64 v[7:8], v0, v[7:8] ; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064_DPP-NEXT: buffer_gl0_inv ; GFX1064_DPP-NEXT: .LBB29_2: ; GFX1064_DPP-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1064_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x24 -; GFX1064_DPP-NEXT: v_readfirstlane_b32 s1, v10 -; GFX1064_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v10, v2 +; GFX1064_DPP-NEXT: v_readfirstlane_b32 s1, v8 +; GFX1064_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v4 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v5 ; GFX1064_DPP-NEXT: s_mov_b32 s7, 0x31016000 -; GFX1064_DPP-NEXT: v_cmp_gt_u64_e32 vcc, s[0:1], v[9:10] -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v10, v10, s1, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v9, v9, s0, vcc +; GFX1064_DPP-NEXT: v_cmp_gt_u64_e32 vcc, s[0:1], v[7:8] +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v8, v8, s1, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v7, v7, s0, vcc ; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1064_DPP-NEXT: buffer_store_dwordx2 v[9:10], off, s[4:7], 0 +; GFX1064_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[4:7], 0 ; GFX1064_DPP-NEXT: s_endpgm ; ; GFX1032_DPP-LABEL: umax_i64_varying: ; GFX1032_DPP: ; %bb.0: ; %entry ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v4, 0, 0, s4 -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s4 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[3:4], v[7:8] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[3:4], v[7:8] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc_lo -; GFX1032_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1032_DPP-NEXT: v_permlanex16_b32 v8, v3, -1, -1 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo -; GFX1032_DPP-NEXT: v_readlane_b32 s1, v4, 31 -; GFX1032_DPP-NEXT: v_readlane_b32 s6, v3, 15 -; GFX1032_DPP-NEXT: v_readlane_b32 s0, v3, 31 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_readlane_b32 s5, v4, 15 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s4 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s4 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[1:2], v[5:6] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[1:2], v[5:6] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX1032_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 +; GFX1032_DPP-NEXT: v_permlanex16_b32 v6, v1, -1, -1 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032_DPP-NEXT: v_readlane_b32 s1, v2, 31 +; GFX1032_DPP-NEXT: v_readlane_b32 s6, v1, 15 +; GFX1032_DPP-NEXT: v_readlane_b32 s0, v1, 31 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_readlane_b32 s5, v2, 15 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4 -; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0 +; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 ; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1032_DPP-NEXT: v_writelane_b32 v2, s5, 16 -; GFX1032_DPP-NEXT: v_writelane_b32 v1, s6, 16 +; GFX1032_DPP-NEXT: v_writelane_b32 v5, s5, 16 +; GFX1032_DPP-NEXT: v_writelane_b32 v4, s6, 16 ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4 ; GFX1032_DPP-NEXT: s_mov_b32 s6, -1 -; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v9 -; GFX1032_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10 +; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7 +; GFX1032_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX1032_DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo ; GFX1032_DPP-NEXT: s_cbranch_execz .LBB29_2 ; GFX1032_DPP-NEXT: ; %bb.1: -; GFX1032_DPP-NEXT: v_mov_b32_e32 v10, s1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, s0 -; GFX1032_DPP-NEXT: ds_max_rtn_u64 v[9:10], v0, v[9:10] +; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, s1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, s0 +; GFX1032_DPP-NEXT: ds_max_rtn_u64 v[7:8], v0, v[7:8] ; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032_DPP-NEXT: buffer_gl0_inv ; GFX1032_DPP-NEXT: .LBB29_2: ; GFX1032_DPP-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1032_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x24 -; GFX1032_DPP-NEXT: v_readfirstlane_b32 s1, v10 -; GFX1032_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v10, v2 +; GFX1032_DPP-NEXT: v_readfirstlane_b32 s1, v8 +; GFX1032_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v4 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v5 ; GFX1032_DPP-NEXT: s_mov_b32 s7, 0x31016000 -; GFX1032_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[9:10] -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v10, v10, s1, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v9, v9, s0, vcc_lo +; GFX1032_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[7:8] +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v8, v8, s1, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v7, v7, s0, vcc_lo ; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1032_DPP-NEXT: buffer_store_dwordx2 v[9:10], off, s[4:7], 0 +; GFX1032_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[4:7], 0 ; GFX1032_DPP-NEXT: s_endpgm ; ; GFX1164_DPP-LABEL: umax_i64_varying: ; GFX1164_DPP: ; %bb.0: ; %entry -; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v4, 0, 0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[3:4], v[5:6] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[0:1] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[3:4], v[7:8] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v2 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[3:4], v[5:6] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[5:6] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[3:4], v[7:8] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1164_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[5:6] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1164_DPP-NEXT: v_permlanex16_b32 v8, v3, -1, -1 +; GFX1164_DPP-NEXT: v_permlanex16_b32 v6, v1, -1, -1 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1164_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v2, 31 +; GFX1164_DPP-NEXT: v_readlane_b32 s5, v1, 31 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[3:4], v[5:6] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v4, 31 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v3, 31 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s4 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, s4 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, s5 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, s5 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1164_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[3:4], v[5:6] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1) +; GFX1164_DPP-NEXT: v_cmp_gt_u64_e32 vcc, v[1:2], v[3:4] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v4, 15 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v3, 15 -; GFX1164_DPP-NEXT: v_readlane_b32 s6, v4, 31 -; GFX1164_DPP-NEXT: v_readlane_b32 s7, v3, 31 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s4, 16 -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v3, 63 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s5, 16 -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v4, 63 -; GFX1164_DPP-NEXT: v_readlane_b32 s8, v4, 47 -; GFX1164_DPP-NEXT: v_readlane_b32 s9, v3, 47 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s6, 32 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s7, 32 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v2, 15 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_readlane_b32 s5, v1, 15 +; GFX1164_DPP-NEXT: v_readlane_b32 s6, v2, 31 +; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 31 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s4, 16 +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v1, 63 +; GFX1164_DPP-NEXT: v_writelane_b32 v4, s5, 16 +; GFX1164_DPP-NEXT: v_readlane_b32 s5, v2, 63 +; GFX1164_DPP-NEXT: v_readlane_b32 s8, v2, 47 +; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 47 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32 +; GFX1164_DPP-NEXT: v_writelane_b32 v4, s7, 32 ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v9, exec_hi, v0 +; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[6:7], -1 ; GFX1164_DPP-NEXT: s_mov_b64 s[0:1], s[4:5] -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s8, 48 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s9, 48 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s8, 48 +; GFX1164_DPP-NEXT: v_writelane_b32 v4, s9, 48 ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7] -; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9 +; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX1164_DPP-NEXT: s_mov_b32 s6, -1 -; GFX1164_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10 +; GFX1164_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GFX1164_DPP-NEXT: s_cbranch_execz .LBB29_2 ; GFX1164_DPP-NEXT: ; %bb.1: -; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, s1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, s0 -; GFX1164_DPP-NEXT: ds_max_rtn_u64 v[9:10], v0, v[9:10] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, s1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s0 +; GFX1164_DPP-NEXT: ds_max_rtn_u64 v[7:8], v0, v[7:8] ; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164_DPP-NEXT: buffer_gl0_inv ; GFX1164_DPP-NEXT: .LBB29_2: ; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1164_DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x24 -; GFX1164_DPP-NEXT: v_readfirstlane_b32 s1, v10 -; GFX1164_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, v2 +; GFX1164_DPP-NEXT: v_readfirstlane_b32 s1, v8 +; GFX1164_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, v4 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v5 ; GFX1164_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_cmp_gt_u64_e32 vcc, s[0:1], v[9:10] -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v10, v10, s1, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v9, v9, s0, vcc +; GFX1164_DPP-NEXT: v_cmp_gt_u64_e32 vcc, s[0:1], v[7:8] +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v8, v8, s1, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v7, v7, s0, vcc ; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1164_DPP-NEXT: buffer_store_b64 v[9:10], off, s[4:7], 0 +; GFX1164_DPP-NEXT: buffer_store_b64 v[7:8], off, s[4:7], 0 ; GFX1164_DPP-NEXT: s_endpgm ; ; GFX1132_DPP-LABEL: umax_i64_varying: ; GFX1132_DPP: ; %bb.0: ; %entry -; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v1, 0 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v4, 0, 0, s4 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v6, v2 -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s4 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v8, v2 +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s4 +; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v3, 0 +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s4 +; GFX1132_DPP-NEXT: v_dual_mov_b32 v6, 0 :: v_dual_mov_b32 v5, 0 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v3, v5, v3 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v6, v2 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v2, v4, v2 :: v_dual_cndmask_b32 v1, v3, v1 +; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v3, 0 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[3:4], v[7:8] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v8, v4 :: v_dual_cndmask_b32 v3, v7, v3 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v8, v2 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[1:2], v[5:6] +; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v2, v6, v2 :: v_dual_cndmask_b32 v1, v5, v1 +; GFX1132_DPP-NEXT: v_dual_mov_b32 v6, 0 :: v_dual_mov_b32 v5, 0 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v3, v5, v3 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v6, v2 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v2, v4, v2 :: v_dual_cndmask_b32 v1, v3, v1 +; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v3, 0 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[3:4], v[7:8] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v8, v4 :: v_dual_cndmask_b32 v3, v7, v3 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[1:2], v[5:6] +; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v2, v6, v2 :: v_dual_cndmask_b32 v1, v5, v1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1132_DPP-NEXT: v_permlanex16_b32 v8, v3, -1, -1 +; GFX1132_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 +; GFX1132_DPP-NEXT: v_permlanex16_b32 v6, v1, -1, -1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v3, v5, v3 -; GFX1132_DPP-NEXT: v_readlane_b32 s1, v4, 31 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_readlane_b32 s6, v3, 15 -; GFX1132_DPP-NEXT: v_readlane_b32 s0, v3, 31 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_readlane_b32 s5, v4, 15 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1132_DPP-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_cndmask_b32 v2, v4, v2 +; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_cndmask_b32 v1, v3, v1 +; GFX1132_DPP-NEXT: v_readlane_b32 s1, v2, 31 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15 +; GFX1132_DPP-NEXT: v_readlane_b32 s0, v1, 31 +; GFX1132_DPP-NEXT: v_readlane_b32 s5, v2, 15 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4 ; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0 +; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 ; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1132_DPP-NEXT: v_writelane_b32 v2, s5, 16 -; GFX1132_DPP-NEXT: v_writelane_b32 v1, s6, 16 +; GFX1132_DPP-NEXT: v_writelane_b32 v5, s5, 16 +; GFX1132_DPP-NEXT: v_writelane_b32 v4, s6, 16 ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4 ; GFX1132_DPP-NEXT: s_mov_b32 s6, -1 -; GFX1132_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v9 -; GFX1132_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10 +; GFX1132_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7 +; GFX1132_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX1132_DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo ; GFX1132_DPP-NEXT: s_cbranch_execz .LBB29_2 ; GFX1132_DPP-NEXT: ; %bb.1: -; GFX1132_DPP-NEXT: v_dual_mov_b32 v10, s1 :: v_dual_mov_b32 v9, s0 -; GFX1132_DPP-NEXT: ds_max_rtn_u64 v[9:10], v0, v[9:10] +; GFX1132_DPP-NEXT: v_dual_mov_b32 v8, s1 :: v_dual_mov_b32 v7, s0 +; GFX1132_DPP-NEXT: ds_max_rtn_u64 v[7:8], v0, v[7:8] ; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132_DPP-NEXT: buffer_gl0_inv ; GFX1132_DPP-NEXT: .LBB29_2: ; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1132_DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x24 -; GFX1132_DPP-NEXT: v_readfirstlane_b32 s1, v10 -; GFX1132_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v9, v1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v10, v2 +; GFX1132_DPP-NEXT: v_readfirstlane_b32 s1, v8 +; GFX1132_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v7, v4 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v8, v5 ; GFX1132_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[9:10] -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v10, v10, s1, vcc_lo -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v9, v9, s0, vcc_lo +; GFX1132_DPP-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[7:8] +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v8, v8, s1, vcc_lo +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v7, v7, s0, vcc_lo ; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1132_DPP-NEXT: buffer_store_b64 v[9:10], off, s[4:7], 0 +; GFX1132_DPP-NEXT: buffer_store_b64 v[7:8], off, s[4:7], 0 ; GFX1132_DPP-NEXT: s_endpgm entry: %lane = call i32 @llvm.amdgcn.workitem.id.x() @@ -17221,585 +17108,579 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) { ; ; GFX8_DPP-LABEL: umin_i64_varying: ; GFX8_DPP: ; %bb.0: ; %entry -; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 -; GFX8_DPP-NEXT: v_mov_b32_e32 v9, 0 -; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v7 +; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0 +; GFX8_DPP-NEXT: v_mov_b32_e32 v8, 0 +; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v6 ; GFX8_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX8_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX8_DPP-NEXT: v_cndmask_b32_e64 v3, -1, 0, s[0:1] +; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, -1, v0, s[0:1] +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX8_DPP-NEXT: v_mov_b32_e32 v1, -1 +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[4:5] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, -1 ; GFX8_DPP-NEXT: v_mov_b32_e32 v4, -1 -; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, -1, 0, s[0:1] -; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s[0:1] -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 -; GFX8_DPP-NEXT: s_mov_b32 s6, -1 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[4:5] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, -1 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[4:5] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, -1 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[4:5] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, -1 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[4:5] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX8_DPP-NEXT: v_mov_b32_e32 v4, -1 ; GFX8_DPP-NEXT: s_nop 0 -; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v6, v2 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX8_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[5:6] -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8_DPP-NEXT: v_readlane_b32 s5, v2, 63 -; GFX8_DPP-NEXT: v_readlane_b32 s4, v1, 63 -; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 wave_shr:1 row_mask:0xf bank_mask:0xf -; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX8_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[4:5] +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v2, -1 +; GFX8_DPP-NEXT: v_readlane_b32 s5, v3, 63 +; GFX8_DPP-NEXT: v_readlane_b32 s4, v4, 63 +; GFX8_DPP-NEXT: v_mov_b32_dpp v2, v3 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX8_DPP-NEXT: v_mov_b32_dpp v1, v4 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX8_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 -; GFX8_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 +; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6 +; GFX8_DPP-NEXT: ; implicit-def: $vgpr6_vgpr7 ; GFX8_DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX8_DPP-NEXT: s_cbranch_execz .LBB32_2 ; GFX8_DPP-NEXT: ; %bb.1: -; GFX8_DPP-NEXT: v_mov_b32_e32 v8, s5 -; GFX8_DPP-NEXT: v_mov_b32_e32 v7, s4 +; GFX8_DPP-NEXT: v_mov_b32_e32 v7, s5 +; GFX8_DPP-NEXT: v_mov_b32_e32 v6, s4 ; GFX8_DPP-NEXT: s_mov_b32 m0, -1 -; GFX8_DPP-NEXT: ds_min_rtn_u64 v[7:8], v9, v[7:8] +; GFX8_DPP-NEXT: ds_min_rtn_u64 v[6:7], v8, v[6:7] ; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX8_DPP-NEXT: .LBB32_2: ; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[0:1] -; GFX8_DPP-NEXT: v_readfirstlane_b32 s1, v8 -; GFX8_DPP-NEXT: v_readfirstlane_b32 s0, v7 -; GFX8_DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX8_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x24 -; GFX8_DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX8_DPP-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[7:8] -; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s1 -; GFX8_DPP-NEXT: v_cndmask_b32_e32 v8, v8, v0, vcc -; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s0 -; GFX8_DPP-NEXT: s_mov_b32 s7, 0xf000 +; GFX8_DPP-NEXT: v_readfirstlane_b32 s5, v7 +; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v6 +; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v1 +; GFX8_DPP-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24 +; GFX8_DPP-NEXT: v_mov_b32_e32 v7, v2 +; GFX8_DPP-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[6:7] +; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s5 ; GFX8_DPP-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc +; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s4 +; GFX8_DPP-NEXT: s_mov_b32 s3, 0xf000 +; GFX8_DPP-NEXT: s_mov_b32 s2, -1 +; GFX8_DPP-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc ; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX8_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[4:7], 0 +; GFX8_DPP-NEXT: buffer_store_dwordx2 v[6:7], off, s[0:3], 0 ; GFX8_DPP-NEXT: s_endpgm ; ; GFX9_DPP-LABEL: umin_i64_varying: ; GFX9_DPP: ; %bb.0: ; %entry -; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 -; GFX9_DPP-NEXT: v_mov_b32_e32 v9, 0 -; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v7 +; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0 +; GFX9_DPP-NEXT: v_mov_b32_e32 v8, 0 +; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v6 ; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX9_DPP-NEXT: v_cndmask_b32_e64 v3, -1, 0, s[0:1] +; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, -1, v0, s[0:1] +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX9_DPP-NEXT: v_mov_b32_e32 v1, -1 +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[4:5] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, -1 ; GFX9_DPP-NEXT: v_mov_b32_e32 v4, -1 -; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, -1, 0, s[0:1] -; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s[0:1] -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 -; GFX9_DPP-NEXT: s_mov_b32 s6, -1 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[4:5] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, -1 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[4:5] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, -1 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[4:5] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, -1 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3 -; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v4 +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[4:5] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX9_DPP-NEXT: v_mov_b32_e32 v4, -1 ; GFX9_DPP-NEXT: s_nop 0 -; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v1 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v6, v2 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[5:6] -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9_DPP-NEXT: v_readlane_b32 s5, v2, 63 -; GFX9_DPP-NEXT: v_readlane_b32 s4, v1, 63 -; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 wave_shr:1 row_mask:0xf bank_mask:0xf -; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[4:5] +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v2, -1 +; GFX9_DPP-NEXT: v_readlane_b32 s5, v3, 63 +; GFX9_DPP-NEXT: v_readlane_b32 s4, v4, 63 +; GFX9_DPP-NEXT: v_mov_b32_dpp v2, v3 wave_shr:1 row_mask:0xf bank_mask:0xf +; GFX9_DPP-NEXT: v_mov_b32_dpp v1, v4 wave_shr:1 row_mask:0xf bank_mask:0xf ; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 -; GFX9_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 +; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6 +; GFX9_DPP-NEXT: ; implicit-def: $vgpr6_vgpr7 ; GFX9_DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX9_DPP-NEXT: s_cbranch_execz .LBB32_2 ; GFX9_DPP-NEXT: ; %bb.1: -; GFX9_DPP-NEXT: v_mov_b32_e32 v8, s5 -; GFX9_DPP-NEXT: v_mov_b32_e32 v7, s4 -; GFX9_DPP-NEXT: ds_min_rtn_u64 v[7:8], v9, v[7:8] +; GFX9_DPP-NEXT: v_mov_b32_e32 v7, s5 +; GFX9_DPP-NEXT: v_mov_b32_e32 v6, s4 +; GFX9_DPP-NEXT: ds_min_rtn_u64 v[6:7], v8, v[6:7] ; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9_DPP-NEXT: .LBB32_2: ; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[0:1] -; GFX9_DPP-NEXT: v_readfirstlane_b32 s1, v8 -; GFX9_DPP-NEXT: v_readfirstlane_b32 s0, v7 -; GFX9_DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX9_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x24 -; GFX9_DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX9_DPP-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[7:8] -; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s1 -; GFX9_DPP-NEXT: v_cndmask_b32_e32 v8, v8, v0, vcc -; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s0 -; GFX9_DPP-NEXT: s_mov_b32 s7, 0xf000 +; GFX9_DPP-NEXT: v_readfirstlane_b32 s5, v7 +; GFX9_DPP-NEXT: v_readfirstlane_b32 s4, v6 +; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v1 +; GFX9_DPP-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24 +; GFX9_DPP-NEXT: v_mov_b32_e32 v7, v2 +; GFX9_DPP-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[6:7] +; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s5 ; GFX9_DPP-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc +; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s4 +; GFX9_DPP-NEXT: s_mov_b32 s3, 0xf000 +; GFX9_DPP-NEXT: s_mov_b32 s2, -1 +; GFX9_DPP-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc ; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX9_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[4:7], 0 +; GFX9_DPP-NEXT: buffer_store_dwordx2 v[6:7], off, s[0:3], 0 ; GFX9_DPP-NEXT: s_endpgm ; ; GFX1064_DPP-LABEL: umin_i64_varying: ; GFX1064_DPP: ; %bb.0: ; %entry ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v1, -1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v2, -1 -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v4, -1, 0, s[0:1] -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v3, -1, v0, s[0:1] -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[3:4], v[5:6] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[3:4], v[7:8] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[3:4], v[5:6] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[3:4], v[7:8] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX1064_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1064_DPP-NEXT: v_permlanex16_b32 v8, v3, -1, -1 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[3:4], v[5:6] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v4, 31 -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v3, 31 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, s4 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s5 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1064_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[3:4], v[5:6] -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v2, -1, 0, s[0:1] +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s[0:1] +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, -1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[3:4] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[5:6] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, -1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[3:4] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[5:6] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GFX1064_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 +; GFX1064_DPP-NEXT: v_permlanex16_b32 v6, v1, -1, -1 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1064_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[3:4] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v2, 31 +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v1, 31 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, s4 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, s5 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX1064_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[3:4] +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX1064_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v4, 15 -; GFX1064_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v3, 15 -; GFX1064_DPP-NEXT: v_readlane_b32 s6, v4, 31 -; GFX1064_DPP-NEXT: v_readlane_b32 s7, v3, 31 -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s4, 16 -; GFX1064_DPP-NEXT: v_readlane_b32 s4, v3, 63 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s5, 16 -; GFX1064_DPP-NEXT: v_readlane_b32 s5, v4, 63 -; GFX1064_DPP-NEXT: v_readlane_b32 s8, v4, 47 -; GFX1064_DPP-NEXT: v_readlane_b32 s9, v3, 47 -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s6, 32 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s7, 32 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v2, 15 +; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v1, 15 +; GFX1064_DPP-NEXT: v_readlane_b32 s6, v2, 31 +; GFX1064_DPP-NEXT: v_readlane_b32 s7, v1, 31 +; GFX1064_DPP-NEXT: v_writelane_b32 v5, s4, 16 +; GFX1064_DPP-NEXT: v_readlane_b32 s4, v1, 63 +; GFX1064_DPP-NEXT: v_writelane_b32 v4, s5, 16 +; GFX1064_DPP-NEXT: v_readlane_b32 s5, v2, 63 +; GFX1064_DPP-NEXT: v_readlane_b32 s8, v2, 47 +; GFX1064_DPP-NEXT: v_readlane_b32 s9, v1, 47 +; GFX1064_DPP-NEXT: v_writelane_b32 v5, s6, 32 +; GFX1064_DPP-NEXT: v_writelane_b32 v4, s7, 32 ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v9, exec_hi, v0 +; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[6:7], -1 ; GFX1064_DPP-NEXT: s_mov_b64 s[0:1], s[4:5] -; GFX1064_DPP-NEXT: v_writelane_b32 v2, s8, 48 -; GFX1064_DPP-NEXT: v_writelane_b32 v1, s9, 48 +; GFX1064_DPP-NEXT: v_writelane_b32 v5, s8, 48 +; GFX1064_DPP-NEXT: v_writelane_b32 v4, s9, 48 ; GFX1064_DPP-NEXT: s_mov_b64 exec, s[6:7] -; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9 +; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX1064_DPP-NEXT: s_mov_b32 s6, -1 -; GFX1064_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10 +; GFX1064_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GFX1064_DPP-NEXT: s_cbranch_execz .LBB32_2 ; GFX1064_DPP-NEXT: ; %bb.1: -; GFX1064_DPP-NEXT: v_mov_b32_e32 v10, s1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, s0 -; GFX1064_DPP-NEXT: ds_min_rtn_u64 v[9:10], v0, v[9:10] +; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s1 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, s0 +; GFX1064_DPP-NEXT: ds_min_rtn_u64 v[7:8], v0, v[7:8] ; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064_DPP-NEXT: buffer_gl0_inv ; GFX1064_DPP-NEXT: .LBB32_2: ; GFX1064_DPP-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1064_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x24 -; GFX1064_DPP-NEXT: v_readfirstlane_b32 s1, v10 -; GFX1064_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, v1 -; GFX1064_DPP-NEXT: v_mov_b32_e32 v10, v2 +; GFX1064_DPP-NEXT: v_readfirstlane_b32 s1, v8 +; GFX1064_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, v4 +; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v5 ; GFX1064_DPP-NEXT: s_mov_b32 s7, 0x31016000 -; GFX1064_DPP-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[9:10] -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v10, v10, s1, vcc -; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v9, v9, s0, vcc +; GFX1064_DPP-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[7:8] +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v8, v8, s1, vcc +; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v7, v7, s0, vcc ; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1064_DPP-NEXT: buffer_store_dwordx2 v[9:10], off, s[4:7], 0 +; GFX1064_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[4:7], 0 ; GFX1064_DPP-NEXT: s_endpgm ; ; GFX1032_DPP-LABEL: umin_i64_varying: ; GFX1032_DPP: ; %bb.0: ; %entry ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, -1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v2, -1 -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v4, -1, 0, s4 -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v3, -1, v0, s4 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[3:4], v[7:8] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo -; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[3:4], v[7:8] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc_lo -; GFX1032_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1032_DPP-NEXT: v_permlanex16_b32 v8, v3, -1, -1 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1032_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo -; GFX1032_DPP-NEXT: v_readlane_b32 s1, v4, 31 -; GFX1032_DPP-NEXT: v_readlane_b32 s6, v3, 15 -; GFX1032_DPP-NEXT: v_readlane_b32 s0, v3, 31 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1032_DPP-NEXT: v_readlane_b32 s5, v4, 15 -; GFX1032_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, -1, 0, s4 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s4 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, -1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[1:2], v[5:6] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, -1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[1:2], v[5:6] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX1032_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 +; GFX1032_DPP-NEXT: v_permlanex16_b32 v6, v1, -1, -1 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX1032_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX1032_DPP-NEXT: v_readlane_b32 s1, v2, 31 +; GFX1032_DPP-NEXT: v_readlane_b32 s6, v1, 15 +; GFX1032_DPP-NEXT: v_readlane_b32 s0, v1, 31 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1032_DPP-NEXT: v_readlane_b32 s5, v2, 15 +; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4 -; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0 +; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 ; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1032_DPP-NEXT: v_writelane_b32 v2, s5, 16 -; GFX1032_DPP-NEXT: v_writelane_b32 v1, s6, 16 +; GFX1032_DPP-NEXT: v_writelane_b32 v5, s5, 16 +; GFX1032_DPP-NEXT: v_writelane_b32 v4, s6, 16 ; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4 ; GFX1032_DPP-NEXT: s_mov_b32 s6, -1 -; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v9 -; GFX1032_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10 +; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7 +; GFX1032_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX1032_DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo ; GFX1032_DPP-NEXT: s_cbranch_execz .LBB32_2 ; GFX1032_DPP-NEXT: ; %bb.1: -; GFX1032_DPP-NEXT: v_mov_b32_e32 v10, s1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, s0 -; GFX1032_DPP-NEXT: ds_min_rtn_u64 v[9:10], v0, v[9:10] +; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, s1 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, s0 +; GFX1032_DPP-NEXT: ds_min_rtn_u64 v[7:8], v0, v[7:8] ; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032_DPP-NEXT: buffer_gl0_inv ; GFX1032_DPP-NEXT: .LBB32_2: ; GFX1032_DPP-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1032_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x24 -; GFX1032_DPP-NEXT: v_readfirstlane_b32 s1, v10 -; GFX1032_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, v1 -; GFX1032_DPP-NEXT: v_mov_b32_e32 v10, v2 +; GFX1032_DPP-NEXT: v_readfirstlane_b32 s1, v8 +; GFX1032_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v4 +; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v5 ; GFX1032_DPP-NEXT: s_mov_b32 s7, 0x31016000 -; GFX1032_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, s[0:1], v[9:10] -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v10, v10, s1, vcc_lo -; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v9, v9, s0, vcc_lo +; GFX1032_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, s[0:1], v[7:8] +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v8, v8, s1, vcc_lo +; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v7, v7, s0, vcc_lo ; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1032_DPP-NEXT: buffer_store_dwordx2 v[9:10], off, s[4:7], 0 +; GFX1032_DPP-NEXT: buffer_store_dwordx2 v[7:8], off, s[4:7], 0 ; GFX1032_DPP-NEXT: s_endpgm ; ; GFX1164_DPP-LABEL: umin_i64_varying: ; GFX1164_DPP: ; %bb.0: ; %entry -; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v1, -1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, -1 -; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v4, -1, 0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v3, -1, v0, s[0:1] -; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v2 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[3:4], v[5:6] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v2, -1, 0, s[0:1] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s[0:1] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, -1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[3:4], v[7:8] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v2 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[3:4] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[3:4], v[5:6] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[5:6] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, -1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[3:4], v[7:8] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[3:4] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1164_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[5:6] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc +; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1164_DPP-NEXT: v_permlanex16_b32 v8, v3, -1, -1 +; GFX1164_DPP-NEXT: v_permlanex16_b32 v6, v1, -1, -1 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1164_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[3:4] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, -1 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v2, 31 +; GFX1164_DPP-NEXT: v_readlane_b32 s5, v1, 31 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4) -; GFX1164_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[3:4], v[5:6] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v4, 31 -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v3, 31 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s4 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, s4 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, s5 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, s5 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf -; GFX1164_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[3:4], v[5:6] -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf +; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, -1 +; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1) +; GFX1164_DPP-NEXT: v_cmp_lt_u64_e32 vcc, v[1:2], v[3:4] +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, -1 +; GFX1164_DPP-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v4, 15 -; GFX1164_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v3, 15 -; GFX1164_DPP-NEXT: v_readlane_b32 s6, v4, 31 -; GFX1164_DPP-NEXT: v_readlane_b32 s7, v3, 31 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s4, 16 -; GFX1164_DPP-NEXT: v_readlane_b32 s4, v3, 63 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s5, 16 -; GFX1164_DPP-NEXT: v_readlane_b32 s5, v4, 63 -; GFX1164_DPP-NEXT: v_readlane_b32 s8, v4, 47 -; GFX1164_DPP-NEXT: v_readlane_b32 s9, v3, 47 -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s6, 32 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s7, 32 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v2, 15 +; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1164_DPP-NEXT: v_readlane_b32 s5, v1, 15 +; GFX1164_DPP-NEXT: v_readlane_b32 s6, v2, 31 +; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 31 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s4, 16 +; GFX1164_DPP-NEXT: v_readlane_b32 s4, v1, 63 +; GFX1164_DPP-NEXT: v_writelane_b32 v4, s5, 16 +; GFX1164_DPP-NEXT: v_readlane_b32 s5, v2, 63 +; GFX1164_DPP-NEXT: v_readlane_b32 s8, v2, 47 +; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 47 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32 +; GFX1164_DPP-NEXT: v_writelane_b32 v4, s7, 32 ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v9, exec_hi, v0 +; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[6:7], -1 ; GFX1164_DPP-NEXT: s_mov_b64 s[0:1], s[4:5] -; GFX1164_DPP-NEXT: v_writelane_b32 v2, s8, 48 -; GFX1164_DPP-NEXT: v_writelane_b32 v1, s9, 48 +; GFX1164_DPP-NEXT: v_writelane_b32 v5, s8, 48 +; GFX1164_DPP-NEXT: v_writelane_b32 v4, s9, 48 ; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7] -; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v9 +; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX1164_DPP-NEXT: s_mov_b32 s6, -1 -; GFX1164_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10 +; GFX1164_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc ; GFX1164_DPP-NEXT: s_cbranch_execz .LBB32_2 ; GFX1164_DPP-NEXT: ; %bb.1: -; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, s1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, s0 -; GFX1164_DPP-NEXT: ds_min_rtn_u64 v[9:10], v0, v[9:10] +; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, s1 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s0 +; GFX1164_DPP-NEXT: ds_min_rtn_u64 v[7:8], v0, v[7:8] ; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164_DPP-NEXT: buffer_gl0_inv ; GFX1164_DPP-NEXT: .LBB32_2: ; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1164_DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x24 -; GFX1164_DPP-NEXT: v_readfirstlane_b32 s1, v10 -; GFX1164_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, v1 -; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, v2 +; GFX1164_DPP-NEXT: v_readfirstlane_b32 s1, v8 +; GFX1164_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, v4 +; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v5 ; GFX1164_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1164_DPP-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[9:10] -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v10, v10, s1, vcc -; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v9, v9, s0, vcc +; GFX1164_DPP-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[7:8] +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v8, v8, s1, vcc +; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v7, v7, s0, vcc ; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1164_DPP-NEXT: buffer_store_b64 v[9:10], off, s[4:7], 0 +; GFX1164_DPP-NEXT: buffer_store_b64 v[7:8], off, s[4:7], 0 ; GFX1164_DPP-NEXT: s_endpgm ; ; GFX1132_DPP-LABEL: umin_i64_varying: ; GFX1132_DPP: ; %bb.0: ; %entry -; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v1, -1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, -1 -; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v4, -1, 0, s4 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v6, v2 -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v3, -1, v0, s4 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v8, v2 +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v2, -1, 0, s4 +; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, -1 :: v_dual_mov_b32 v3, -1 +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s4 +; GFX1132_DPP-NEXT: v_dual_mov_b32 v6, -1 :: v_dual_mov_b32 v5, -1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v3, v5, v3 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v6, v2 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v2, v4, v2 :: v_dual_cndmask_b32 v1, v3, v1 +; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, -1 :: v_dual_mov_b32 v3, -1 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:2 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[3:4], v[7:8] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v8, v4 :: v_dual_cndmask_b32 v3, v7, v3 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v8, v2 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[1:2], v[5:6] +; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v2, v6, v2 :: v_dual_cndmask_b32 v1, v5, v1 +; GFX1132_DPP-NEXT: v_dual_mov_b32 v6, -1 :: v_dual_mov_b32 v5, -1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v3, v5, v3 -; GFX1132_DPP-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v6, v2 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v2, v4, v2 :: v_dual_cndmask_b32 v1, v3, v1 +; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, -1 :: v_dual_mov_b32 v3, -1 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:8 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[3:4], v[7:8] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v8, v4 :: v_dual_cndmask_b32 v3, v7, v3 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[1:2], v[5:6] +; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v2, v6, v2 :: v_dual_cndmask_b32 v1, v5, v1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_permlanex16_b32 v7, v4, -1, -1 -; GFX1132_DPP-NEXT: v_permlanex16_b32 v8, v3, -1, -1 +; GFX1132_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1 +; GFX1132_DPP-NEXT: v_permlanex16_b32 v6, v1, -1, -1 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v7 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v8 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[3:4], v[5:6] -; GFX1132_DPP-NEXT: v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v3, v5, v3 -; GFX1132_DPP-NEXT: v_readlane_b32 s1, v4, 31 -; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1132_DPP-NEXT: v_readlane_b32 s6, v3, 15 -; GFX1132_DPP-NEXT: v_readlane_b32 s0, v3, 31 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX1132_DPP-NEXT: v_readlane_b32 s5, v4, 15 -; GFX1132_DPP-NEXT: v_mov_b32_dpp v1, v3 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v5 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[1:2], v[3:4] +; GFX1132_DPP-NEXT: v_dual_mov_b32 v5, -1 :: v_dual_cndmask_b32 v2, v4, v2 +; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, -1 :: v_dual_cndmask_b32 v1, v3, v1 +; GFX1132_DPP-NEXT: v_readlane_b32 s1, v2, 31 +; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15 +; GFX1132_DPP-NEXT: v_readlane_b32 s0, v1, 31 +; GFX1132_DPP-NEXT: v_readlane_b32 s5, v2, 15 +; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4 ; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0 +; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 ; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1 -; GFX1132_DPP-NEXT: v_writelane_b32 v2, s5, 16 -; GFX1132_DPP-NEXT: v_writelane_b32 v1, s6, 16 +; GFX1132_DPP-NEXT: v_writelane_b32 v5, s5, 16 +; GFX1132_DPP-NEXT: v_writelane_b32 v4, s6, 16 ; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4 ; GFX1132_DPP-NEXT: s_mov_b32 s6, -1 -; GFX1132_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v9 -; GFX1132_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10 +; GFX1132_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7 +; GFX1132_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8 ; GFX1132_DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo ; GFX1132_DPP-NEXT: s_cbranch_execz .LBB32_2 ; GFX1132_DPP-NEXT: ; %bb.1: -; GFX1132_DPP-NEXT: v_dual_mov_b32 v10, s1 :: v_dual_mov_b32 v9, s0 -; GFX1132_DPP-NEXT: ds_min_rtn_u64 v[9:10], v0, v[9:10] +; GFX1132_DPP-NEXT: v_dual_mov_b32 v8, s1 :: v_dual_mov_b32 v7, s0 +; GFX1132_DPP-NEXT: ds_min_rtn_u64 v[7:8], v0, v[7:8] ; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132_DPP-NEXT: buffer_gl0_inv ; GFX1132_DPP-NEXT: .LBB32_2: ; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1132_DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x24 -; GFX1132_DPP-NEXT: v_readfirstlane_b32 s1, v10 -; GFX1132_DPP-NEXT: v_readfirstlane_b32 s0, v9 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v9, v1 -; GFX1132_DPP-NEXT: v_mov_b32_e32 v10, v2 +; GFX1132_DPP-NEXT: v_readfirstlane_b32 s1, v8 +; GFX1132_DPP-NEXT: v_readfirstlane_b32 s0, v7 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v7, v4 +; GFX1132_DPP-NEXT: v_mov_b32_e32 v8, v5 ; GFX1132_DPP-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1132_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, s[0:1], v[9:10] -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v10, v10, s1, vcc_lo -; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v9, v9, s0, vcc_lo +; GFX1132_DPP-NEXT: v_cmp_lt_u64_e32 vcc_lo, s[0:1], v[7:8] +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v8, v8, s1, vcc_lo +; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v7, v7, s0, vcc_lo ; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1132_DPP-NEXT: buffer_store_b64 v[9:10], off, s[4:7], 0 +; GFX1132_DPP-NEXT: buffer_store_b64 v[7:8], off, s[4:7], 0 ; GFX1132_DPP-NEXT: s_endpgm entry: %lane = call i32 @llvm.amdgcn.workitem.id.x() diff --git a/llvm/test/CodeGen/AMDGPU/dpp64_combine.ll b/llvm/test/CodeGen/AMDGPU/dpp64_combine.ll index 60ce730c3eed33..c7422786d344ee 100644 --- a/llvm/test/CodeGen/AMDGPU/dpp64_combine.ll +++ b/llvm/test/CodeGen/AMDGPU/dpp64_combine.ll @@ -1,7 +1,7 @@ ; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GCN,DPP64,GFX90A ; RUN: llc -mtriple=amdgcn -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GCN,DPP64,DPPMOV64,GFX940 -; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GCN,DPP32,GFX10PLUS,GFX10 -; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GCN,DPP32,GFX10PLUS,GFX11 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GCN,DPP32,GFX10PLUS +; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GCN,DPP32,GFX10PLUS ; GCN-LABEL: {{^}}dpp64_ceil: ; GCN: global_load_{{dwordx2|b64}} [[V:v\[[0-9:]+\]]], @@ -75,16 +75,14 @@ define amdgpu_kernel void @dpp64_div(ptr addrspace(1) %arg, i64 %in1) { ; GFX90A: v_add_co_u32_e32 ; GFX90A: v_addc_co_u32_e32 ; GFX940: v_lshl_add_u64 -; GFX10: v_mov_b32_dpp -; GFX10: v_add_co_u32 -; GFX10: v_add_co_ci_u32_e32 -; GFX11: v_add_co_u32_e64_dpp -; GFX11: v_add_co_ci_u32_e32 -define amdgpu_cs void @dpp64_loop(i64 %arg) { +; GFX10PLUS: v_mov_b32_dpp +; GFX10PLUS: v_add_co_u32 +; GFX10PLUS: v_add_co_ci_u32_e32 +define amdgpu_cs void @dpp64_loop(i64 %arg, i64 %val) { bb: br label %bb1 bb1: - %i = call i64 @llvm.amdgcn.update.dpp.i64(i64 0, i64 0, i32 0, i32 0, i32 0, i1 false) + %i = call i64 @llvm.amdgcn.update.dpp.i64(i64 poison, i64 %val, i32 0, i32 0, i32 0, i1 false) %i2 = add i64 %i, %arg %i3 = atomicrmw add ptr addrspace(1) null, i64 %i2 monotonic, align 8 br label %bb1 diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll index fa0ab4537fcd76..7cc78b47a72214 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll @@ -7307,58 +7307,56 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9-DPP-NEXT: v_mov_b32_e32 v10, 0 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v11, 1 -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v9, v8, v1, s[0:1] -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v8, 0, v0, s[0:1] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 -; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v11, v9, v1, s[0:1] +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] ; GFX9-DPP-NEXT: s_nop 1 -; GFX9-DPP-NEXT: v_mov_b32_dpp v10, v8 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v11, v9 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX9-DPP-NEXT: v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1 ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX9-DPP-NEXT: v_readlane_b32 s43, v9, 63 ; GFX9-DPP-NEXT: v_readlane_b32 s42, v8, 63 ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX9-DPP-NEXT: s_cbranch_execz .LBB10_3 ; GFX9-DPP-NEXT: ; %bb.1: ; GFX9-DPP-NEXT: s_load_dwordx2 s[44:45], s[36:37], 0x24 -; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DPP-NEXT: s_mov_b64 s[46:47], 0 ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[44:45] @@ -7439,28 +7437,28 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1] ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1064-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1064-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 ; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] @@ -7473,13 +7471,13 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v8 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v42, v9 -; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 -; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX1064-DPP-NEXT: s_cbranch_execz .LBB10_3 ; GFX1064-DPP-NEXT: ; %bb.1: ; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[34:35], 0x24 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[42:43] @@ -7561,42 +7559,42 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v11, 0x80000000, v1, s0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s0 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1032-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1032-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 ; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, v8 -; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v42, v9 ; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 -; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 ; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX1032-DPP-NEXT: s_cbranch_execz .LBB10_3 ; GFX1032-DPP-NEXT: ; %bb.1: ; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[34:35], 0x24 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[42:43] ; GFX1032-DPP-NEXT: .LBB10_2: ; %atomicrmw.start @@ -7667,35 +7665,34 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1] ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1164-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 @@ -7709,16 +7706,16 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, v8 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mov_b32_e32 v42, v9 ; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec ; GFX1164-DPP-NEXT: s_waitcnt_depctr 0xfffe -; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 -; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1 ; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3 ; GFX1164-DPP-NEXT: ; %bb.1: ; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[34:35], 0x24 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: global_load_b64 v[1:2], v0, s[42:43] @@ -7789,49 +7786,50 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v11, 0x80000000, v1, s0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] ; GFX1132-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1132-DPP-NEXT: v_mov_b32_e32 v41, v8 -; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v41, v8 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX1132-DPP-NEXT: v_mov_b32_e32 v42, v9 ; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 ; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1 ; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3 ; GFX1132-DPP-NEXT: ; %bb.1: ; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[34:35], 0x24 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v0, s[42:43] ; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 @@ -8885,59 +8883,57 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v6, 1 -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v1, s[0:1] -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 -; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1] +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] ; GFX9-DPP-NEXT: s_nop 1 -; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v4 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4] ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1 ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63 ; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63 ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3] -; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc ; GFX9-DPP-NEXT: s_cbranch_execz .LBB12_3 ; GFX9-DPP-NEXT: ; %bb.1: ; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24 -; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0 ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3] @@ -8985,28 +8981,28 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1] ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] ; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0 ; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] @@ -9017,27 +9013,27 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a ; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5] ; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4 -; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX1064-DPP-NEXT: s_cbranch_execz .LBB12_3 ; GFX1064-DPP-NEXT: ; %bb.1: ; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1064-DPP-NEXT: global_load_dwordx2 v[11:12], v2, s[0:1] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1] ; GFX1064-DPP-NEXT: .LBB12_2: ; %atomicrmw.start ; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1064-DPP-NEXT: v_add_f64 v[9:10], v[11:12], v[0:1] -; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[9:10], v2, v[9:12], s[0:1] glc +; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc ; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[9:10], v[11:12] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v9 +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7 ; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB12_2 @@ -9073,53 +9069,53 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] ; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0 ; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] ; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3 -; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4 ; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 -; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7 ; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX1032-DPP-NEXT: s_cbranch_execz .LBB12_3 ; GFX1032-DPP-NEXT: ; %bb.1: ; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1032-DPP-NEXT: global_load_dwordx2 v[11:12], v2, s[0:1] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1] ; GFX1032-DPP-NEXT: .LBB12_2: ; %atomicrmw.start ; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1032-DPP-NEXT: v_add_f64 v[9:10], v[11:12], v[0:1] -; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[9:10], v2, v[9:12], s[0:1] glc +; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc ; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[9:10], v[11:12] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v11, v9 +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7 ; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 ; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB12_2 @@ -9145,35 +9141,34 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1] ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0 @@ -9184,32 +9179,31 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a ; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2 ; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec ; GFX1164-DPP-NEXT: s_waitcnt_depctr 0xfffe -; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v0 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v8 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6 ; GFX1164-DPP-NEXT: s_cbranch_execz .LBB12_3 ; GFX1164-DPP-NEXT: ; %bb.1: ; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1164-DPP-NEXT: global_load_b64 v[10:11], v12, s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1] ; GFX1164-DPP-NEXT: .LBB12_2: ; %atomicrmw.start ; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[0:1] -; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[8:9], v12, v[8:11], s[0:1] glc +; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc ; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, v9 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6 ; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] @@ -9236,59 +9230,60 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] ; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2 -; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v8, exec_lo, 0 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v3 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3 ; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 ; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v8 +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6 ; GFX1132-DPP-NEXT: s_cbranch_execz .LBB12_3 ; GFX1132-DPP-NEXT: ; %bb.1: ; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1132-DPP-NEXT: global_load_b64 v[10:11], v12, s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1] ; GFX1132-DPP-NEXT: .LBB12_2: ; %atomicrmw.start ; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[0:1] -; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[8:9], v12, v[8:11], s[0:1] glc +; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc ; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[8:9], v[10:11] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, v9 :: v_dual_mov_b32 v10, v8 +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6 ; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 @@ -10309,59 +10304,57 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v6, 1 -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v1, s[0:1] -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 -; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1] +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] ; GFX9-DPP-NEXT: s_nop 1 -; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v4 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4] ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1 ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63 ; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63 ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3] -; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc ; GFX9-DPP-NEXT: s_cbranch_execz .LBB14_3 ; GFX9-DPP-NEXT: ; %bb.1: ; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24 -; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0 ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3] @@ -10409,28 +10402,28 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1] ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] ; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0 ; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] @@ -10441,27 +10434,27 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5] ; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4 -; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX1064-DPP-NEXT: s_cbranch_execz .LBB14_3 ; GFX1064-DPP-NEXT: ; %bb.1: ; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1064-DPP-NEXT: global_load_dwordx2 v[11:12], v2, s[0:1] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1] ; GFX1064-DPP-NEXT: .LBB14_2: ; %atomicrmw.start ; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1064-DPP-NEXT: v_add_f64 v[9:10], v[11:12], v[0:1] -; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[9:10], v2, v[9:12], s[0:1] glc +; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc ; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[9:10], v[11:12] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v9 +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7 ; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB14_2 @@ -10497,53 +10490,53 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] ; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0 ; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] ; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3 -; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4 ; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 -; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7 ; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX1032-DPP-NEXT: s_cbranch_execz .LBB14_3 ; GFX1032-DPP-NEXT: ; %bb.1: ; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1032-DPP-NEXT: global_load_dwordx2 v[11:12], v2, s[0:1] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1] ; GFX1032-DPP-NEXT: .LBB14_2: ; %atomicrmw.start ; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1032-DPP-NEXT: v_add_f64 v[9:10], v[11:12], v[0:1] -; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[9:10], v2, v[9:12], s[0:1] glc +; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc ; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[9:10], v[11:12] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v11, v9 +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7 ; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 ; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB14_2 @@ -10569,35 +10562,34 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1] ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0 @@ -10608,32 +10600,31 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2 ; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec ; GFX1164-DPP-NEXT: s_waitcnt_depctr 0xfffe -; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v0 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v8 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6 ; GFX1164-DPP-NEXT: s_cbranch_execz .LBB14_3 ; GFX1164-DPP-NEXT: ; %bb.1: ; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1164-DPP-NEXT: global_load_b64 v[10:11], v12, s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1] ; GFX1164-DPP-NEXT: .LBB14_2: ; %atomicrmw.start ; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[0:1] -; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[8:9], v12, v[8:11], s[0:1] glc +; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc ; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, v9 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6 ; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] @@ -10660,59 +10651,60 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] ; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2 -; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v8, exec_lo, 0 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v3 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3 ; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 ; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v8 +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6 ; GFX1132-DPP-NEXT: s_cbranch_execz .LBB14_3 ; GFX1132-DPP-NEXT: ; %bb.1: ; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1132-DPP-NEXT: global_load_b64 v[10:11], v12, s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1] ; GFX1132-DPP-NEXT: .LBB14_2: ; %atomicrmw.start ; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[0:1] -; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[8:9], v12, v[8:11], s[0:1] glc +; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc ; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[8:9], v[10:11] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, v9 :: v_dual_mov_b32 v10, v8 +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6 ; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 @@ -11215,59 +11207,57 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v6, 1 -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v1, s[0:1] -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 -; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1] +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] ; GFX9-DPP-NEXT: s_nop 1 -; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v4 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4] ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1 ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63 ; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63 ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3] -; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc ; GFX9-DPP-NEXT: s_cbranch_execz .LBB15_3 ; GFX9-DPP-NEXT: ; %bb.1: ; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24 -; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0 ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3] @@ -11315,28 +11305,28 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1] ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] ; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0 ; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] @@ -11347,27 +11337,27 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5] ; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4 -; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX1064-DPP-NEXT: s_cbranch_execz .LBB15_3 ; GFX1064-DPP-NEXT: ; %bb.1: ; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1064-DPP-NEXT: global_load_dwordx2 v[11:12], v2, s[0:1] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1] ; GFX1064-DPP-NEXT: .LBB15_2: ; %atomicrmw.start ; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1064-DPP-NEXT: v_add_f64 v[9:10], v[11:12], v[0:1] -; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[9:10], v2, v[9:12], s[0:1] glc +; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc ; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[9:10], v[11:12] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v9 +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7 ; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB15_2 @@ -11403,53 +11393,53 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] ; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0 ; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] ; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3 -; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4 ; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 -; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7 ; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX1032-DPP-NEXT: s_cbranch_execz .LBB15_3 ; GFX1032-DPP-NEXT: ; %bb.1: ; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1032-DPP-NEXT: global_load_dwordx2 v[11:12], v2, s[0:1] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1] ; GFX1032-DPP-NEXT: .LBB15_2: ; %atomicrmw.start ; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1032-DPP-NEXT: v_add_f64 v[9:10], v[11:12], v[0:1] -; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[9:10], v2, v[9:12], s[0:1] glc +; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc ; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[9:10], v[11:12] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v11, v9 +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7 ; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 ; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB15_2 @@ -11475,35 +11465,34 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1] ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0 @@ -11514,32 +11503,31 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2 ; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec ; GFX1164-DPP-NEXT: s_waitcnt_depctr 0xfffe -; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v0 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v8 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6 ; GFX1164-DPP-NEXT: s_cbranch_execz .LBB15_3 ; GFX1164-DPP-NEXT: ; %bb.1: ; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1164-DPP-NEXT: global_load_b64 v[10:11], v12, s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1] ; GFX1164-DPP-NEXT: .LBB15_2: ; %atomicrmw.start ; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[0:1] -; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[8:9], v12, v[8:11], s[0:1] glc +; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc ; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, v9 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6 ; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] @@ -11566,59 +11554,60 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] ; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2 -; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v8, exec_lo, 0 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v3 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3 ; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 ; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v8 +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6 ; GFX1132-DPP-NEXT: s_cbranch_execz .LBB15_3 ; GFX1132-DPP-NEXT: ; %bb.1: ; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1132-DPP-NEXT: global_load_b64 v[10:11], v12, s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1] ; GFX1132-DPP-NEXT: .LBB15_2: ; %atomicrmw.start ; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[0:1] -; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[8:9], v12, v[8:11], s[0:1] glc +; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc ; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[8:9], v[10:11] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, v9 :: v_dual_mov_b32 v10, v8 +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6 ; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 @@ -13286,58 +13275,56 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9-DPP-NEXT: v_mov_b32_e32 v10, 0 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v11, 1 -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v9, v8, v1, s[0:1] -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v8, 0, v0, s[0:1] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v11, v9, v1, s[0:1] +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 -; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] ; GFX9-DPP-NEXT: s_nop 1 -; GFX9-DPP-NEXT: v_mov_b32_dpp v10, v8 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v11, v9 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX9-DPP-NEXT: v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1 ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX9-DPP-NEXT: v_readlane_b32 s43, v9, 63 ; GFX9-DPP-NEXT: v_readlane_b32 s42, v8, 63 ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX9-DPP-NEXT: s_cbranch_execz .LBB17_3 ; GFX9-DPP-NEXT: ; %bb.1: ; GFX9-DPP-NEXT: s_load_dwordx2 s[44:45], s[36:37], 0x24 -; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DPP-NEXT: s_mov_b64 s[46:47], 0 ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[44:45] @@ -13418,28 +13405,28 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1] ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1064-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1064-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 ; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] @@ -13452,13 +13439,13 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau ; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v8 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v42, v9 -; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 -; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX1064-DPP-NEXT: s_cbranch_execz .LBB17_3 ; GFX1064-DPP-NEXT: ; %bb.1: ; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[34:35], 0x24 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[42:43] @@ -13540,42 +13527,42 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v11, 0x80000000, v1, s0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s0 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1032-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1032-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 ; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, v8 -; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v42, v9 ; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 -; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 ; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX1032-DPP-NEXT: s_cbranch_execz .LBB17_3 ; GFX1032-DPP-NEXT: ; %bb.1: ; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[34:35], 0x24 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[42:43] ; GFX1032-DPP-NEXT: .LBB17_2: ; %atomicrmw.start @@ -13646,35 +13633,34 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1] ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1164-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 @@ -13688,16 +13674,16 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau ; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, v8 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mov_b32_e32 v42, v9 ; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec ; GFX1164-DPP-NEXT: s_waitcnt_depctr 0xfffe -; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 -; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1 ; GFX1164-DPP-NEXT: s_cbranch_execz .LBB17_3 ; GFX1164-DPP-NEXT: ; %bb.1: ; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[34:35], 0x24 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: global_load_b64 v[1:2], v0, s[42:43] @@ -13768,49 +13754,50 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v11, 0x80000000, v1, s0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] ; GFX1132-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1132-DPP-NEXT: v_mov_b32_e32 v41, v8 -; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v41, v8 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX1132-DPP-NEXT: v_mov_b32_e32 v42, v9 ; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 ; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1 ; GFX1132-DPP-NEXT: s_cbranch_execz .LBB17_3 ; GFX1132-DPP-NEXT: ; %bb.1: ; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[34:35], 0x24 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v0, s[42:43] ; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll index 9b9dd744945b15..8e7181a0cf4495 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll @@ -4709,65 +4709,63 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9-DPP-NEXT: v_mov_b32_e32 v10, 0 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000 -; GFX9-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v9, v8, v1, s[0:1] -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v8, 0, v0, s[0:1] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 -; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] +; GFX9-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v11, v9, v1, s[0:1] +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[12:13] +; GFX9-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] ; GFX9-DPP-NEXT: s_nop 1 -; GFX9-DPP-NEXT: v_mov_b32_dpp v10, v8 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v11, v9 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX9-DPP-NEXT: v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] +; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[8:9] ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1 ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX9-DPP-NEXT: v_readlane_b32 s43, v9, 63 ; GFX9-DPP-NEXT: v_readlane_b32 s42, v8, 63 ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX9-DPP-NEXT: s_cbranch_execz .LBB7_3 ; GFX9-DPP-NEXT: ; %bb.1: ; GFX9-DPP-NEXT: s_load_dwordx2 s[44:45], s[36:37], 0x24 -; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DPP-NEXT: s_mov_b64 s[46:47], 0 ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[44:45] @@ -4850,33 +4848,33 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1] ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX1064-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] ; GFX1064-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[8:9] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1064-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1064-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1064-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] ; GFX1064-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1064-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 ; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] @@ -4892,13 +4890,13 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent ; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v8 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v42, v9 -; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 -; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX1064-DPP-NEXT: s_cbranch_execz .LBB7_3 ; GFX1064-DPP-NEXT: ; %bb.1: ; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[34:35], 0x24 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[42:43] @@ -4982,48 +4980,48 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s0 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX1032-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] ; GFX1032-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[8:9] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1032-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1032-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1032-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] ; GFX1032-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1032-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 ; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] ; GFX1032-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] ; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v8 -; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v9 ; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 -; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 ; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX1032-DPP-NEXT: s_cbranch_execz .LBB7_3 ; GFX1032-DPP-NEXT: ; %bb.1: ; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[34:35], 0x24 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032-DPP-NEXT: v_max_f64 v[41:42], v[3:4], v[3:4] ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[42:43] @@ -5096,42 +5094,41 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1] ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 +; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] +; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[8:9] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 +; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] -; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[8:9] +; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1164-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 @@ -5148,16 +5145,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent ; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v8 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v9 ; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec -; GFX1164-DPP-NEXT: s_waitcnt_depctr 0xfffe -; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 -; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1 ; GFX1164-DPP-NEXT: s_cbranch_execz .LBB7_3 ; GFX1164-DPP-NEXT: ; %bb.1: ; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[34:35], 0x24 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1164-DPP-NEXT: v_max_f64 v[41:42], v[3:4], v[3:4] ; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) @@ -5230,57 +5227,55 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v8, 0 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, 0x7ff80000 :: v_dual_mov_b32 v8, 0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX1132-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[8:9] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1132-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1132-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1132-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] ; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, v8 -; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v8 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 ; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, v9 ; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 ; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo -; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1 ; GFX1132-DPP-NEXT: s_cbranch_execz .LBB7_3 ; GFX1132-DPP-NEXT: ; %bb.1: ; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[34:35], 0x24 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1132-DPP-NEXT: v_max_f64 v[41:42], v[3:4], v[3:4] ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v0, s[42:43] @@ -6155,66 +6150,64 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0 -; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000 -; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000 -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v1, s[0:1] -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 -; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000 +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1] +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] ; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[7:8] +; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8] ; GFX9-DPP-NEXT: s_nop 1 -; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v4 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] -; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6] +; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4] +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4] ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1 ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63 ; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63 ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3] -; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc ; GFX9-DPP-NEXT: s_cbranch_execz .LBB9_3 ; GFX9-DPP-NEXT: ; %bb.1: ; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24 -; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0 ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3] @@ -6264,33 +6257,33 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1] ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] -; GFX1064-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4] ; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6] ; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0 ; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] @@ -6304,15 +6297,15 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a ; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4] ; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4 -; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX1064-DPP-NEXT: s_cbranch_execz .LBB9_2 ; GFX1064-DPP-NEXT: ; %bb.1: ; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[0:1] ; GFX1064-DPP-NEXT: .LBB9_2: @@ -6347,47 +6340,47 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] -; GFX1032-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4] ; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6] ; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0 ; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] ; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6] ; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3 -; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4 -; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7 ; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX1032-DPP-NEXT: s_cbranch_execz .LBB9_2 ; GFX1032-DPP-NEXT: ; %bb.1: ; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[0:1] ; GFX1032-DPP-NEXT: .LBB9_2: @@ -6412,42 +6405,41 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1] ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[6:7] -; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 +; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[6:7] -; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 +; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] +; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[6:7] -; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] +; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3] -; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3] +; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] +; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0 @@ -6461,35 +6453,34 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a ; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] ; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5] ; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec -; GFX1164-DPP-NEXT: s_waitcnt_depctr 0xfffe -; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v0 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v8 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6 ; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3 ; GFX1164-DPP-NEXT: ; %bb.1: ; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, 0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1] ; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1164-DPP-NEXT: global_load_b64 v[10:11], v12, s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1] ; GFX1164-DPP-NEXT: .LBB9_2: ; %atomicrmw.start ; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[10:11] +; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[0:1] -; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[8:9], v12, v[8:11], s[0:1] glc +; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc ; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, v9 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6 ; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] @@ -6516,70 +6507,68 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] -; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[6:7] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[6:7] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[6:7] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[6:7] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[6:7] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[6:7] -; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] +; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] +; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] +; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5] ; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2 -; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v8, exec_lo, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v3 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3 ; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 ; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo -; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v8 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6 ; GFX1132-DPP-NEXT: s_cbranch_execz .LBB9_3 ; GFX1132-DPP-NEXT: ; %bb.1: ; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1] ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1132-DPP-NEXT: global_load_b64 v[10:11], v12, s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1] ; GFX1132-DPP-NEXT: .LBB9_2: ; %atomicrmw.start ; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1132-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[10:11] +; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9] ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[0:1] -; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[8:9], v12, v[8:11], s[0:1] glc +; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc ; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[8:9], v[10:11] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, v9 :: v_dual_mov_b32 v10, v8 +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6 ; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 @@ -8205,65 +8194,63 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9-DPP-NEXT: v_mov_b32_e32 v10, 0 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000 -; GFX9-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v9, v8, v1, s[0:1] -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v8, 0, v0, s[0:1] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 -; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] +; GFX9-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v11, v9, v1, s[0:1] +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[12:13] +; GFX9-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] ; GFX9-DPP-NEXT: s_nop 1 -; GFX9-DPP-NEXT: v_mov_b32_dpp v10, v8 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v11, v9 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX9-DPP-NEXT: v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] +; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[8:9] ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1 ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX9-DPP-NEXT: v_readlane_b32 s43, v9, 63 ; GFX9-DPP-NEXT: v_readlane_b32 s42, v8, 63 ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX9-DPP-NEXT: s_cbranch_execz .LBB11_3 ; GFX9-DPP-NEXT: ; %bb.1: ; GFX9-DPP-NEXT: s_load_dwordx2 s[44:45], s[36:37], 0x24 -; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DPP-NEXT: s_mov_b64 s[46:47], 0 ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[44:45] @@ -8346,33 +8333,33 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1] ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX1064-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] ; GFX1064-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[8:9] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1064-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1064-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1064-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] ; GFX1064-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1064-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 ; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] @@ -8388,13 +8375,13 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau ; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v8 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v42, v9 -; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 -; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX1064-DPP-NEXT: s_cbranch_execz .LBB11_3 ; GFX1064-DPP-NEXT: ; %bb.1: ; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[34:35], 0x24 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[42:43] @@ -8478,48 +8465,48 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s0 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX1032-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] ; GFX1032-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[8:9] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1032-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1032-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1032-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] ; GFX1032-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1032-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 ; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] ; GFX1032-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] ; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v8 -; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v9 ; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 -; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 ; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_3 ; GFX1032-DPP-NEXT: ; %bb.1: ; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[34:35], 0x24 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032-DPP-NEXT: v_max_f64 v[41:42], v[3:4], v[3:4] ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[42:43] @@ -8592,42 +8579,41 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1] ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 +; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] +; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[8:9] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 +; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] -; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[8:9] +; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1164-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 @@ -8644,16 +8630,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau ; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v8 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v9 ; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec -; GFX1164-DPP-NEXT: s_waitcnt_depctr 0xfffe -; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 -; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1 ; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3 ; GFX1164-DPP-NEXT: ; %bb.1: ; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[34:35], 0x24 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1164-DPP-NEXT: v_max_f64 v[41:42], v[3:4], v[3:4] ; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) @@ -8726,57 +8712,55 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v8, 0 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, 0x7ff80000 :: v_dual_mov_b32 v8, 0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX1132-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[8:9] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1132-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1132-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1132-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[10:11] ; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, v8 -; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v8 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 ; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, v9 ; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 ; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo -; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1 ; GFX1132-DPP-NEXT: s_cbranch_execz .LBB11_3 ; GFX1132-DPP-NEXT: ; %bb.1: ; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[34:35], 0x24 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1132-DPP-NEXT: v_max_f64 v[41:42], v[3:4], v[3:4] ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v0, s[42:43] diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll index 21a65851db1d55..b95b52168625dd 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll @@ -4709,65 +4709,63 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9-DPP-NEXT: v_mov_b32_e32 v10, 0 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000 -; GFX9-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v9, v8, v1, s[0:1] -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v8, 0, v0, s[0:1] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 -; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] +; GFX9-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v11, v9, v1, s[0:1] +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[12:13] +; GFX9-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] ; GFX9-DPP-NEXT: s_nop 1 -; GFX9-DPP-NEXT: v_mov_b32_dpp v10, v8 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v11, v9 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX9-DPP-NEXT: v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] +; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[10:11], v[8:9] ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1 ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX9-DPP-NEXT: v_readlane_b32 s43, v9, 63 ; GFX9-DPP-NEXT: v_readlane_b32 s42, v8, 63 ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX9-DPP-NEXT: s_cbranch_execz .LBB7_3 ; GFX9-DPP-NEXT: ; %bb.1: ; GFX9-DPP-NEXT: s_load_dwordx2 s[44:45], s[36:37], 0x24 -; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DPP-NEXT: s_mov_b64 s[46:47], 0 ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[44:45] @@ -4850,33 +4848,33 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1] ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX1064-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1064-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1064-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1064-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] ; GFX1064-DPP-NEXT: v_min_f64 v[8:9], v[10:11], v[8:9] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1064-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1064-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1064-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] ; GFX1064-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1064-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 ; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] @@ -4892,13 +4890,13 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent ; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v8 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v42, v9 -; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 -; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX1064-DPP-NEXT: s_cbranch_execz .LBB7_3 ; GFX1064-DPP-NEXT: ; %bb.1: ; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[34:35], 0x24 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[42:43] @@ -4982,48 +4980,48 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s0 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX1032-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1032-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1032-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1032-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] ; GFX1032-DPP-NEXT: v_min_f64 v[8:9], v[10:11], v[8:9] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1032-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1032-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1032-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] ; GFX1032-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1032-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 ; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] ; GFX1032-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] ; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v8 -; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v9 ; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 -; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 ; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX1032-DPP-NEXT: s_cbranch_execz .LBB7_3 ; GFX1032-DPP-NEXT: ; %bb.1: ; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[34:35], 0x24 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032-DPP-NEXT: v_max_f64 v[41:42], v[3:4], v[3:4] ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[42:43] @@ -5096,42 +5094,41 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1] ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1164-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 +; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] +; GFX1164-DPP-NEXT: v_min_f64 v[8:9], v[10:11], v[8:9] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1164-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 +; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1164-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1164-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1164-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] -; GFX1164-DPP-NEXT: v_min_f64 v[8:9], v[10:11], v[8:9] +; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1164-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1164-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 @@ -5148,16 +5145,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent ; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v8 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v9 ; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec -; GFX1164-DPP-NEXT: s_waitcnt_depctr 0xfffe -; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 -; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1 ; GFX1164-DPP-NEXT: s_cbranch_execz .LBB7_3 ; GFX1164-DPP-NEXT: ; %bb.1: ; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[34:35], 0x24 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1164-DPP-NEXT: v_max_f64 v[41:42], v[3:4], v[3:4] ; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) @@ -5230,57 +5227,55 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v8, 0 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, 0x7ff80000 :: v_dual_mov_b32 v8, 0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX1132-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_min_f64 v[8:9], v[10:11], v[8:9] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1132-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1132-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1132-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] ; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, v8 -; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v8 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 ; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, v9 ; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 ; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo -; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1 ; GFX1132-DPP-NEXT: s_cbranch_execz .LBB7_3 ; GFX1132-DPP-NEXT: ; %bb.1: ; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[34:35], 0x24 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1132-DPP-NEXT: v_max_f64 v[41:42], v[3:4], v[3:4] ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v0, s[42:43] @@ -6155,66 +6150,64 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0 -; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000 -; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000 -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v1, s[0:1] -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 -; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000 +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1] +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] ; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[7:8] +; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8] ; GFX9-DPP-NEXT: s_nop 1 -; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v4 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] -; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6] +; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4] +; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4] ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1 ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63 ; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63 ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3] -; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc ; GFX9-DPP-NEXT: s_cbranch_execz .LBB9_3 ; GFX9-DPP-NEXT: ; %bb.1: ; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24 -; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0 ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3] @@ -6264,33 +6257,33 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1] ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] -; GFX1064-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX1064-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX1064-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX1064-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4] ; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] +; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] +; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] +; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6] ; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0 ; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] @@ -6304,15 +6297,15 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a ; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4] ; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4 -; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX1064-DPP-NEXT: s_cbranch_execz .LBB9_2 ; GFX1064-DPP-NEXT: ; %bb.1: ; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[0:1] ; GFX1064-DPP-NEXT: .LBB9_2: @@ -6347,47 +6340,47 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] -; GFX1032-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX1032-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX1032-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8] -; GFX1032-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4] ; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] +; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] +; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] +; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6] ; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0 ; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6] ; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6] ; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3 -; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4 -; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7 ; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX1032-DPP-NEXT: s_cbranch_execz .LBB9_2 ; GFX1032-DPP-NEXT: ; %bb.1: ; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[0:1] ; GFX1032-DPP-NEXT: .LBB9_2: @@ -6412,42 +6405,41 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1] ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[6:7] -; GFX1164-DPP-NEXT: v_min_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 +; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[6:7] -; GFX1164-DPP-NEXT: v_min_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 +; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] +; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[6:7] -; GFX1164-DPP-NEXT: v_min_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] +; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3] -; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3] +; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] +; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0 @@ -6461,35 +6453,34 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a ; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] ; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5] ; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec -; GFX1164-DPP-NEXT: s_waitcnt_depctr 0xfffe -; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v0 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v8 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6 ; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3 ; GFX1164-DPP-NEXT: ; %bb.1: ; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, 0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1] ; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1164-DPP-NEXT: global_load_b64 v[10:11], v12, s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1] ; GFX1164-DPP-NEXT: .LBB9_2: ; %atomicrmw.start ; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[10:11] +; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[0:1] -; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[8:9], v12, v[8:11], s[0:1] glc +; GFX1164-DPP-NEXT: v_min_f64 v[6:7], v[6:7], v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc ; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, v9 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6 ; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] @@ -6516,70 +6507,68 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] -; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[6:7] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_min_f64 v[4:5], v[4:5], v[6:7] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[6:7] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_min_f64 v[4:5], v[4:5], v[6:7] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[6:7] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_min_f64 v[4:5], v[4:5], v[6:7] -; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] +; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] +; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] +; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5] ; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2 -; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v8, exec_lo, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v3 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3 ; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 ; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo -; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v8 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6 ; GFX1132-DPP-NEXT: s_cbranch_execz .LBB9_3 ; GFX1132-DPP-NEXT: ; %bb.1: ; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1] ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1132-DPP-NEXT: global_load_b64 v[10:11], v12, s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1] ; GFX1132-DPP-NEXT: .LBB9_2: ; %atomicrmw.start ; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1132-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[10:11] +; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9] ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[0:1] -; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[8:9], v12, v[8:11], s[0:1] glc +; GFX1132-DPP-NEXT: v_min_f64 v[6:7], v[6:7], v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc ; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[8:9], v[10:11] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, v9 :: v_dual_mov_b32 v10, v8 +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6 ; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 @@ -8205,65 +8194,63 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9-DPP-NEXT: v_mov_b32_e32 v10, 0 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000 -; GFX9-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v9, v8, v1, s[0:1] -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v8, 0, v0, s[0:1] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 -; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] +; GFX9-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v11, v9, v1, s[0:1] +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_e32 v13, 0x7ff80000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf ; GFX9-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[12:13] +; GFX9-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] ; GFX9-DPP-NEXT: s_nop 1 -; GFX9-DPP-NEXT: v_mov_b32_dpp v10, v8 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v11, v9 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX9-DPP-NEXT: v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] +; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[10:11], v[8:9] ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1 ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX9-DPP-NEXT: v_readlane_b32 s43, v9, 63 ; GFX9-DPP-NEXT: v_readlane_b32 s42, v8, 63 ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX9-DPP-NEXT: s_cbranch_execz .LBB11_3 ; GFX9-DPP-NEXT: ; %bb.1: ; GFX9-DPP-NEXT: s_load_dwordx2 s[44:45], s[36:37], 0x24 -; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DPP-NEXT: s_mov_b64 s[46:47], 0 ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[44:45] @@ -8346,33 +8333,33 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1] ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX1064-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1064-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1064-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1064-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] ; GFX1064-DPP-NEXT: v_min_f64 v[8:9], v[10:11], v[8:9] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1064-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1064-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1064-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] ; GFX1064-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1064-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 ; GFX1064-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] @@ -8388,13 +8375,13 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau ; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v8 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v42, v9 -; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 -; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX1064-DPP-NEXT: s_cbranch_execz .LBB11_3 ; GFX1064-DPP-NEXT: ; %bb.1: ; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[34:35], 0x24 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[42:43] @@ -8478,48 +8465,48 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s0 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX1032-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1032-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1032-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1032-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] ; GFX1032-DPP-NEXT: v_min_f64 v[8:9], v[10:11], v[8:9] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1032-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1032-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1032-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] ; GFX1032-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1032-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 ; GFX1032-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] ; GFX1032-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] ; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v8 -; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v9 ; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 -; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 ; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_3 ; GFX1032-DPP-NEXT: ; %bb.1: ; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[34:35], 0x24 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032-DPP-NEXT: v_max_f64 v[41:42], v[3:4], v[3:4] ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[42:43] @@ -8592,42 +8579,41 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1] ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1164-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 +; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] +; GFX1164-DPP-NEXT: v_min_f64 v[8:9], v[10:11], v[8:9] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1164-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 +; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1164-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1164-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1164-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0x7ff80000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] -; GFX1164-DPP-NEXT: v_min_f64 v[8:9], v[10:11], v[8:9] +; GFX1164-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1164-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1164-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 @@ -8644,16 +8630,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau ; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v8 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v9 ; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec -; GFX1164-DPP-NEXT: s_waitcnt_depctr 0xfffe -; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 -; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1 ; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3 ; GFX1164-DPP-NEXT: ; %bb.1: ; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[34:35], 0x24 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1164-DPP-NEXT: v_max_f64 v[41:42], v[3:4], v[3:4] ; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) @@ -8726,57 +8712,55 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v8, 0 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v9, 0x7ff80000 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, 0x7ff80000 :: v_dual_mov_b32 v8, 0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX1132-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_max_f64 v[12:13], v[12:13], v[12:13] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_min_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[8:9] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_min_f64 v[8:9], v[10:11], v[8:9] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1132-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1132-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] +; GFX1132-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_max_f64 v[10:11], v[10:11], v[10:11] -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[10:11] ; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, v8 -; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v8 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 ; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, v9 ; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 ; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo -; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1 ; GFX1132-DPP-NEXT: s_cbranch_execz .LBB11_3 ; GFX1132-DPP-NEXT: ; %bb.1: ; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[34:35], 0x24 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1132-DPP-NEXT: v_max_f64 v[41:42], v[3:4], v[3:4] ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v0, s[42:43] diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll index fcd5d0dc497e67..ae4ca7b7356ef9 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll @@ -7635,58 +7635,56 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9-DPP-NEXT: v_mov_b32_e32 v10, 0 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v11, 1 -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v9, v8, v1, s[0:1] -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v8, 0, v0, s[0:1] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 -; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v11, v9, v1, s[0:1] +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] ; GFX9-DPP-NEXT: s_nop 1 -; GFX9-DPP-NEXT: v_mov_b32_dpp v10, v8 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v11, v9 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX9-DPP-NEXT: v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1 ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX9-DPP-NEXT: v_readlane_b32 s43, v9, 63 ; GFX9-DPP-NEXT: v_readlane_b32 s42, v8, 63 ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX9-DPP-NEXT: s_cbranch_execz .LBB10_3 ; GFX9-DPP-NEXT: ; %bb.1: ; GFX9-DPP-NEXT: s_load_dwordx2 s[44:45], s[36:37], 0x24 -; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DPP-NEXT: s_mov_b64 s[46:47], 0 ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[44:45] @@ -7767,28 +7765,28 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1] ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1064-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1064-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 ; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] @@ -7801,13 +7799,13 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v8 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v42, v9 -; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 -; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX1064-DPP-NEXT: s_cbranch_execz .LBB10_3 ; GFX1064-DPP-NEXT: ; %bb.1: ; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[34:35], 0x24 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[42:43] @@ -7889,42 +7887,42 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v11, 0x80000000, v1, s0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s0 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1032-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1032-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 ; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, v8 -; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v42, v9 ; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 -; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 ; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX1032-DPP-NEXT: s_cbranch_execz .LBB10_3 ; GFX1032-DPP-NEXT: ; %bb.1: ; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[34:35], 0x24 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[42:43] ; GFX1032-DPP-NEXT: .LBB10_2: ; %atomicrmw.start @@ -7995,35 +7993,34 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1] ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1164-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 @@ -8037,16 +8034,16 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, v8 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mov_b32_e32 v42, v9 ; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec ; GFX1164-DPP-NEXT: s_waitcnt_depctr 0xfffe -; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 -; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1 ; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3 ; GFX1164-DPP-NEXT: ; %bb.1: ; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[34:35], 0x24 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: global_load_b64 v[1:2], v0, s[42:43] @@ -8117,49 +8114,50 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v11, 0x80000000, v1, s0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] ; GFX1132-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1132-DPP-NEXT: v_mov_b32_e32 v41, v8 -; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v41, v8 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX1132-DPP-NEXT: v_mov_b32_e32 v42, v9 ; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 ; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1 ; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3 ; GFX1132-DPP-NEXT: ; %bb.1: ; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[34:35], 0x24 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v0, s[42:43] ; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 @@ -9212,59 +9210,57 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v6, 1 -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v1, s[0:1] -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 -; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1] +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] ; GFX9-DPP-NEXT: s_nop 1 -; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v4 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4] ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1 ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63 ; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63 ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3] -; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc ; GFX9-DPP-NEXT: s_cbranch_execz .LBB12_3 ; GFX9-DPP-NEXT: ; %bb.1: ; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24 -; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0 ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3] @@ -9312,28 +9308,28 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1] ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] ; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0 ; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] @@ -9344,27 +9340,27 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a ; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5] ; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4 -; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX1064-DPP-NEXT: s_cbranch_execz .LBB12_3 ; GFX1064-DPP-NEXT: ; %bb.1: ; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1064-DPP-NEXT: global_load_dwordx2 v[11:12], v2, s[0:1] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1] ; GFX1064-DPP-NEXT: .LBB12_2: ; %atomicrmw.start ; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1064-DPP-NEXT: v_add_f64 v[9:10], v[11:12], -v[0:1] -; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[9:10], v2, v[9:12], s[0:1] glc +; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc ; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[9:10], v[11:12] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v9 +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7 ; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB12_2 @@ -9400,53 +9396,53 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] ; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0 ; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] ; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3 -; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4 ; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 -; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7 ; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX1032-DPP-NEXT: s_cbranch_execz .LBB12_3 ; GFX1032-DPP-NEXT: ; %bb.1: ; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1032-DPP-NEXT: global_load_dwordx2 v[11:12], v2, s[0:1] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1] ; GFX1032-DPP-NEXT: .LBB12_2: ; %atomicrmw.start ; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1032-DPP-NEXT: v_add_f64 v[9:10], v[11:12], -v[0:1] -; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[9:10], v2, v[9:12], s[0:1] glc +; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc ; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[9:10], v[11:12] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v11, v9 +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7 ; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 ; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB12_2 @@ -9472,35 +9468,34 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1] ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0 @@ -9511,32 +9506,31 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a ; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2 ; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec ; GFX1164-DPP-NEXT: s_waitcnt_depctr 0xfffe -; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v0 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v8 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6 ; GFX1164-DPP-NEXT: s_cbranch_execz .LBB12_3 ; GFX1164-DPP-NEXT: ; %bb.1: ; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1164-DPP-NEXT: global_load_b64 v[10:11], v12, s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1] ; GFX1164-DPP-NEXT: .LBB12_2: ; %atomicrmw.start ; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -v[0:1] -; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[8:9], v12, v[8:11], s[0:1] glc +; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc ; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, v9 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6 ; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] @@ -9563,59 +9557,60 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] ; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2 -; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v8, exec_lo, 0 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v3 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3 ; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 ; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v8 +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6 ; GFX1132-DPP-NEXT: s_cbranch_execz .LBB12_3 ; GFX1132-DPP-NEXT: ; %bb.1: ; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1132-DPP-NEXT: global_load_b64 v[10:11], v12, s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1] ; GFX1132-DPP-NEXT: .LBB12_2: ; %atomicrmw.start ; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -v[0:1] -; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[8:9], v12, v[8:11], s[0:1] glc +; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc ; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[8:9], v[10:11] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, v9 :: v_dual_mov_b32 v10, v8 +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6 ; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 @@ -10636,59 +10631,57 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v6, 1 -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v1, s[0:1] -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 -; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1] +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] ; GFX9-DPP-NEXT: s_nop 1 -; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v4 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4] ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1 ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63 ; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63 ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3] -; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc ; GFX9-DPP-NEXT: s_cbranch_execz .LBB14_3 ; GFX9-DPP-NEXT: ; %bb.1: ; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24 -; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0 ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3] @@ -10736,28 +10729,28 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1] ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] ; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0 ; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] @@ -10768,27 +10761,27 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5] ; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4 -; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX1064-DPP-NEXT: s_cbranch_execz .LBB14_3 ; GFX1064-DPP-NEXT: ; %bb.1: ; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1064-DPP-NEXT: global_load_dwordx2 v[11:12], v2, s[0:1] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1] ; GFX1064-DPP-NEXT: .LBB14_2: ; %atomicrmw.start ; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1064-DPP-NEXT: v_add_f64 v[9:10], v[11:12], -v[0:1] -; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[9:10], v2, v[9:12], s[0:1] glc +; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc ; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[9:10], v[11:12] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v9 +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7 ; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB14_2 @@ -10824,53 +10817,53 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] ; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0 ; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] ; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3 -; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4 ; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 -; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7 ; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX1032-DPP-NEXT: s_cbranch_execz .LBB14_3 ; GFX1032-DPP-NEXT: ; %bb.1: ; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1032-DPP-NEXT: global_load_dwordx2 v[11:12], v2, s[0:1] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1] ; GFX1032-DPP-NEXT: .LBB14_2: ; %atomicrmw.start ; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1032-DPP-NEXT: v_add_f64 v[9:10], v[11:12], -v[0:1] -; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[9:10], v2, v[9:12], s[0:1] glc +; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc ; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[9:10], v[11:12] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v11, v9 +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7 ; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 ; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB14_2 @@ -10896,35 +10889,34 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1] ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0 @@ -10935,32 +10927,31 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2 ; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec ; GFX1164-DPP-NEXT: s_waitcnt_depctr 0xfffe -; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v0 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v8 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6 ; GFX1164-DPP-NEXT: s_cbranch_execz .LBB14_3 ; GFX1164-DPP-NEXT: ; %bb.1: ; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1164-DPP-NEXT: global_load_b64 v[10:11], v12, s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1] ; GFX1164-DPP-NEXT: .LBB14_2: ; %atomicrmw.start ; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -v[0:1] -; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[8:9], v12, v[8:11], s[0:1] glc +; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc ; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, v9 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6 ; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] @@ -10987,59 +10978,60 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] ; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2 -; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v8, exec_lo, 0 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v3 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3 ; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 ; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v8 +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6 ; GFX1132-DPP-NEXT: s_cbranch_execz .LBB14_3 ; GFX1132-DPP-NEXT: ; %bb.1: ; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1132-DPP-NEXT: global_load_b64 v[10:11], v12, s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1] ; GFX1132-DPP-NEXT: .LBB14_2: ; %atomicrmw.start ; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -v[0:1] -; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[8:9], v12, v[8:11], s[0:1] glc +; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc ; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[8:9], v[10:11] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, v9 :: v_dual_mov_b32 v10, v8 +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6 ; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 @@ -11542,59 +11534,57 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v6, 1 -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v1, s[0:1] -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 -; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1] +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] -; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5 -; GFX9-DPP-NEXT: v_mov_b32_e32 v8, v6 +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v3 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v4 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[7:8] +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] ; GFX9-DPP-NEXT: s_nop 1 -; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v4 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4] ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1 ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63 ; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63 ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3] -; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc ; GFX9-DPP-NEXT: s_cbranch_execz .LBB15_3 ; GFX9-DPP-NEXT: ; %bb.1: ; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24 -; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0 ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3] @@ -11642,28 +11632,28 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1] ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] ; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0 ; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] @@ -11674,27 +11664,27 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5] ; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1] ; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4 -; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7 ; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX1064-DPP-NEXT: s_cbranch_execz .LBB15_3 ; GFX1064-DPP-NEXT: ; %bb.1: ; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1064-DPP-NEXT: global_load_dwordx2 v[11:12], v2, s[0:1] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1] ; GFX1064-DPP-NEXT: .LBB15_2: ; %atomicrmw.start ; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1064-DPP-NEXT: v_add_f64 v[9:10], v[11:12], -v[0:1] -; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[9:10], v2, v[9:12], s[0:1] glc +; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc ; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[9:10], v[11:12] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v9 +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7 ; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] ; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB15_2 @@ -11730,53 +11720,53 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, v4 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v6 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8] -; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] ; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0 ; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6] ; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3 -; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4 ; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 -; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7 ; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX1032-DPP-NEXT: s_cbranch_execz .LBB15_3 ; GFX1032-DPP-NEXT: ; %bb.1: ; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1032-DPP-NEXT: global_load_dwordx2 v[11:12], v2, s[0:1] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1] ; GFX1032-DPP-NEXT: .LBB15_2: ; %atomicrmw.start ; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1032-DPP-NEXT: v_add_f64 v[9:10], v[11:12], -v[0:1] -; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[9:10], v2, v[9:12], s[0:1] glc +; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc ; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[9:10], v[11:12] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v11, v9 +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7 ; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 ; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB15_2 @@ -11802,35 +11792,34 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1] ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, v2 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, v3 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 ; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0 @@ -11841,32 +11830,31 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2 ; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec ; GFX1164-DPP-NEXT: s_waitcnt_depctr 0xfffe -; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v0 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v8 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6 ; GFX1164-DPP-NEXT: s_cbranch_execz .LBB15_3 ; GFX1164-DPP-NEXT: ; %bb.1: ; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1164-DPP-NEXT: global_load_b64 v[10:11], v12, s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1] ; GFX1164-DPP-NEXT: .LBB15_2: ; %atomicrmw.start ; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -v[0:1] -; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[8:9], v12, v[8:11], s[0:1] glc +; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc ; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, v9 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8 +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6 ; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] @@ -11893,59 +11881,60 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v3 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v7, v5 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[4:5], v[4:5], v[6:7] +; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3] ; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5] ; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2 -; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v8, exec_lo, 0 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v3 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3 ; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 ; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v8 +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6 ; GFX1132-DPP-NEXT: s_cbranch_execz .LBB15_3 ; GFX1132-DPP-NEXT: ; %bb.1: ; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) -; GFX1132-DPP-NEXT: global_load_b64 v[10:11], v12, s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1] ; GFX1132-DPP-NEXT: .LBB15_2: ; %atomicrmw.start ; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -v[0:1] -; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[8:9], v12, v[8:11], s[0:1] glc +; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc ; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) -; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[8:9], v[10:11] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v11, v9 :: v_dual_mov_b32 v10, v8 +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6 ; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 ; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 @@ -13612,58 +13601,56 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX9-DPP-NEXT: v_mov_b32_e32 v10, 0 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1 -; GFX9-DPP-NEXT: v_bfrev_b32_e32 v11, 1 -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v9, v8, v1, s[0:1] -; GFX9-DPP-NEXT: v_cndmask_b32_e64 v8, 0, v0, s[0:1] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 -; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:1 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v11, v9, v1, s[0:1] +; GFX9-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:2 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:4 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_shr:8 row_mask:0xf bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] -; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v10 -; GFX9-DPP-NEXT: v_mov_b32_e32 v13, v11 +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX9-DPP-NEXT: v_bfrev_b32_e32 v13, 1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v12, 0 ; GFX9-DPP-NEXT: s_nop 0 -; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v8 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v9 row_bcast:15 row_mask:0xa bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[12:13] +; GFX9-DPP-NEXT: v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] ; GFX9-DPP-NEXT: s_nop 1 -; GFX9-DPP-NEXT: v_mov_b32_dpp v10, v8 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_mov_b32_dpp v11, v9 row_bcast:31 row_mask:0xc bank_mask:0xf -; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX9-DPP-NEXT: v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf +; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 -; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1 ; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 ; GFX9-DPP-NEXT: v_readlane_b32 s43, v9, 63 ; GFX9-DPP-NEXT: v_readlane_b32 s42, v8, 63 ; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1] -; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX9-DPP-NEXT: s_cbranch_execz .LBB17_3 ; GFX9-DPP-NEXT: ; %bb.1: ; GFX9-DPP-NEXT: s_load_dwordx2 s[44:45], s[36:37], 0x24 -; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-DPP-NEXT: s_mov_b64 s[46:47], 0 ; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[44:45] @@ -13744,28 +13731,28 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1] ; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1064-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1064-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1064-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1064-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1064-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 ; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] @@ -13778,13 +13765,13 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau ; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v8 ; GFX1064-DPP-NEXT: v_mov_b32_e32 v42, v9 -; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 -; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 ; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc ; GFX1064-DPP-NEXT: s_cbranch_execz .LBB17_3 ; GFX1064-DPP-NEXT: ; %bb.1: ; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[34:35], 0x24 -; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 ; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[42:43] @@ -13866,42 +13853,42 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v11, 0x80000000, v1, s0 ; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s0 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1032-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1032-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1032-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1032-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1032-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 ; GFX1032-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, v8 -; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032-DPP-NEXT: v_mov_b32_e32 v42, v9 ; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 -; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 ; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX1032-DPP-NEXT: s_cbranch_execz .LBB17_3 ; GFX1032-DPP-NEXT: ; %bb.1: ; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[34:35], 0x24 -; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v0, s[42:43] ; GFX1032-DPP-NEXT: .LBB17_2: ; %atomicrmw.start @@ -13972,35 +13959,34 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1] ; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s[0:1] -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_e32 v12, v8 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v13, v9 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1164-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1164-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1164-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1164-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 ; GFX1164-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 @@ -14014,16 +14000,16 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau ; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 ; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, v8 -; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2) ; GFX1164-DPP-NEXT: v_mov_b32_e32 v42, v9 ; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec ; GFX1164-DPP-NEXT: s_waitcnt_depctr 0xfffe -; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 -; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1 ; GFX1164-DPP-NEXT: s_cbranch_execz .LBB17_3 ; GFX1164-DPP-NEXT: ; %bb.1: ; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[34:35], 0x24 -; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 ; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-DPP-NEXT: global_load_b64 v[1:2], v0, s[42:43] @@ -14094,49 +14080,50 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v9, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v8, 0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v11, 0x80000000, v1, s0 ; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v10, 0, v0, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:1 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:2 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] -; GFX1132-DPP-NEXT: v_dual_mov_b32 v12, v8 :: v_dual_mov_b32 v13, v9 -; GFX1132-DPP-NEXT: v_mov_b32_dpp v12, v10 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] +; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v11, 1 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0 +; GFX1132-DPP-NEXT: v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v13, v11 row_xmask:4 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_add_f64 v[10:11], v[10:11], v[12:13] +; GFX1132-DPP-NEXT: v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf +; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_mov_b32_dpp v8, v10 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: v_mov_b32_dpp v9, v11 row_xmask:8 row_mask:0xf bank_mask:0xf -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[10:11], v[8:9] ; GFX1132-DPP-NEXT: v_permlanex16_b32 v11, v9, 0, 0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_permlanex16_b32 v10, v8, 0, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1132-DPP-NEXT: v_add_f64 v[8:9], v[8:9], v[10:11] ; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0 -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX1132-DPP-NEXT: v_mov_b32_e32 v41, v8 -; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v41, v8 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX1132-DPP-NEXT: v_mov_b32_e32 v42, v9 ; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 ; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo -; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1 ; GFX1132-DPP-NEXT: s_cbranch_execz .LBB17_3 ; GFX1132-DPP-NEXT: ; %bb.1: ; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[34:35], 0x24 -; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0 ; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v0, s[42:43] ; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.gfx90a.ll new file mode 100644 index 00000000000000..a3d789c1ccc36f --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.gfx90a.ll @@ -0,0 +1,276 @@ +; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A %s +; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A %s +; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GCN,GFX942 %s +; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GCN,GFX942 %s + +; DPP control value 337 is valid for 64-bit DPP on gfx942 + +; GCN-LABEL: update_dpp_i64: +; +; GFX90A-DAG: v_mov_b32_dpp v2, v0 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; GFX90A-DAG: v_mov_b32_dpp v3, v1 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GFX942: v_mov_b64_dpp v[2:3], v[0:1] row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GCN: global_store_dwordx2 v[4:5], v[2:3], off + +define amdgpu_ps void @update_dpp_i64(i64 %in, i64 %old, ptr addrspace(1) %out) { + %tmp0 = call i64 @llvm.amdgcn.update.dpp.i64(i64 %old, i64 %in, i32 337, i32 1, i32 1, i1 0) + store i64 %tmp0, ptr addrspace(1) %out + ret void +} + +; GCN-LABEL: update_dpp_v2i32: +; +; GFX90A-DAG: v_mov_b32_dpp v2, v0 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; GFX90A-DAG: v_mov_b32_dpp v3, v1 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GFX942: v_mov_b64_dpp v[2:3], v[0:1] row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GCN: global_store_dwordx2 v[4:5], v[2:3], off + +define amdgpu_ps void @update_dpp_v2i32(<2 x i32> %in, <2 x i32> %old, ptr addrspace(1) %out) { + %tmp0 = call <2 x i32> @llvm.amdgcn.update.dpp.v2i32(<2 x i32> %old, <2 x i32> %in, i32 337, i32 1, i32 1, i1 0) + store <2 x i32> %tmp0, ptr addrspace(1) %out + ret void +} + +; GCN-LABEL: update_dpp_v3i32: +; +; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v2 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v1 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v0 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GCN: global_store_dwordx3 + +define amdgpu_ps void @update_dpp_v3i32(<3 x i32> %in, <3 x i32> %old, ptr addrspace(1) %out) { + %tmp0 = call <3 x i32> @llvm.amdgcn.update.dpp.v3i32(<3 x i32> %old, <3 x i32> %in, i32 337, i32 1, i32 1, i1 0) + store <3 x i32> %tmp0, ptr addrspace(1) %out + ret void +} + +; GCN-LABEL: update_dpp_v4i32: +; +; GFX90A-DAG: v_mov_b32_dpp v6, v2 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; GFX90A-DAG: v_mov_b32_dpp v7, v3 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; GFX90A-DAG: v_mov_b32_dpp v4, v0 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; GFX90A-DAG: v_mov_b32_dpp v5, v1 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GFX942-DAG: v_mov_b64_dpp v[6:7], v[2:3] row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; GFX942-DAG: v_mov_b64_dpp v[4:5], v[0:1] row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GCN: global_store_dwordx4 v[8:9], v[4:7], off + +define amdgpu_ps void @update_dpp_v4i32(<4 x i32> %in, <4 x i32> %old, ptr addrspace(1) %out) { + %tmp0 = call <4 x i32> @llvm.amdgcn.update.dpp.v4i32(<4 x i32> %old, <4 x i32> %in, i32 337, i32 1, i32 1, i1 0) + store <4 x i32> %tmp0, ptr addrspace(1) %out + ret void +} + +; GCN-LABEL: update_dpp_v2i32_poison: +; +; GFX90A-DAG: v_mov_b32_dpp v0, v0 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; GFX90A-DAG: v_mov_b32_dpp v1, v1 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GFX942: v_mov_b64_dpp v[0:1], v[0:1] row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GCN: global_store_dwordx2 v[2:3], v[0:1], off + +define amdgpu_ps void @update_dpp_v2i32_poison(<2 x i32> %in, ptr addrspace(1) %out) { + %tmp0 = call <2 x i32> @llvm.amdgcn.update.dpp.v2i32(<2 x i32> poison, <2 x i32> %in, i32 337, i32 1, i32 1, i1 0) + store <2 x i32> %tmp0, ptr addrspace(1) %out + ret void +} + +; GCN-LABEL: update_dpp_float: +; +; GCN: v_mov_b32_dpp v1, v0 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GCN: global_store_dword v[2:3], v1, off + +define amdgpu_ps void @update_dpp_float(float %in, float %old, ptr addrspace(1) %out) { + %tmp0 = call float @llvm.amdgcn.update.dpp.f32(float %old, float %in, i32 337, i32 1, i32 1, i1 0) + store float %tmp0, ptr addrspace(1) %out + ret void +} + +; GCN-LABEL: update_dpp_double: +; +; GFX90A-DAG: v_mov_b32_dpp v2, v0 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; GFX90A-DAG: v_mov_b32_dpp v3, v1 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GFX942: v_mov_b64_dpp v[2:3], v[0:1] row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GCN: global_store_dwordx2 v[4:5], v[2:3], off + +define amdgpu_ps void @update_dpp_double(double %in, double %old, ptr addrspace(1) %out) { + %tmp0 = call double @llvm.amdgcn.update.dpp.f64(double %old, double %in, i32 337, i32 1, i32 1, i1 0) + store double %tmp0, ptr addrspace(1) %out + ret void +} + +; GCN-LABEL: update_dpp_float_poison: +; +; GCN: v_mov_b32_dpp v0, v0 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GCN: global_store_dword v[{{[0-9:]+}}], v0, off + +define amdgpu_ps void @update_dpp_float_poison(float %in, ptr addrspace(1) %out) { + %tmp0 = call float @llvm.amdgcn.update.dpp.f32(float poison, float %in, i32 337, i32 1, i32 1, i1 0) + store float %tmp0, ptr addrspace(1) %out + ret void +} + +; GCN-LABEL: update_dpp_double_poison: +; +; GFX90A-DAG: v_mov_b32_dpp v0, v0 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; GFX90A-DAG: v_mov_b32_dpp v1, v1 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GFX942: v_mov_b64_dpp v[0:1], v[0:1] row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GCN: global_store_dwordx2 v[2:3], v[0:1], off + +define amdgpu_ps void @update_dpp_double_poison(double %in, ptr addrspace(1) %out) { + %tmp0 = call double @llvm.amdgcn.update.dpp.f64(double poison, double %in, i32 337, i32 1, i32 1, i1 0) + store double %tmp0, ptr addrspace(1) %out + ret void +} + +; GCN-LABEL: update_dpp_v2f32_poison: +; +; GFX90A-DAG: v_mov_b32_dpp v0, v0 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; GFX90A-DAG: v_mov_b32_dpp v1, v1 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GFX942: v_mov_b64_dpp v[0:1], v[0:1] row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GCN: global_store_dwordx2 v[2:3], v[0:1], off + +define amdgpu_ps void @update_dpp_v2f32_poison(<2 x float> %in, ptr addrspace(1) %out) { + %tmp0 = call <2 x float> @llvm.amdgcn.update.dpp.v2f32(<2 x float> poison, <2 x float> %in, i32 337, i32 1, i32 1, i1 0) + store <2 x float> %tmp0, ptr addrspace(1) %out + ret void +} + +; GCN-LABEL: update_dpp_v2f16_poison: +; +; GCN: v_mov_b32_dpp v0, v0 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GCN: global_store_dword v[{{[0-9:]+}}], v0, off + +define amdgpu_ps void @update_dpp_v2f16_poison(<2 x half> %in, ptr addrspace(1) %out) { + %tmp0 = call <2 x half> @llvm.amdgcn.update.dpp.v2f16(<2 x half> poison, <2 x half> %in, i32 337, i32 1, i32 1, i1 0) + store <2 x half> %tmp0, ptr addrspace(1) %out + ret void +} + +; GCN-LABEL: update_dpp_v8f16_poison: +; +; GFX90A-DAG: v_mov_b32_dpp v0, v0 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; GFX90A-DAG: v_mov_b32_dpp v1, v1 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; GFX90A-DAG: v_mov_b32_dpp v2, v2 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; GFX90A-DAG: v_mov_b32_dpp v3, v3 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GFX942-DAG: v_mov_b64_dpp v[2:3], v[2:3] row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; GFX942-DAG: v_mov_b64_dpp v[0:1], v[0:1] row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GCN: global_store_dwordx4 v[4:5], v[0:3], off + +define amdgpu_ps void @update_dpp_v8f16_poison(<8 x half> %in, ptr addrspace(1) %out) { + %tmp0 = call <8 x half> @llvm.amdgcn.update.dpp.v8f16(<8 x half> poison, <8 x half> %in, i32 337, i32 1, i32 1, i1 0) + store <8 x half> %tmp0, ptr addrspace(1) %out + ret void +} + +; GCN-LABEL: update_dpp_p3_poison: +; +; GCN: v_mov_b32_dpp v0, v0 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GCN: global_store_dword v[{{[0-9:]+}}], v0, off + +define amdgpu_ps void @update_dpp_p3_poison(ptr addrspace(3) %in, ptr addrspace(1) %out) { + %tmp0 = call ptr addrspace(3) @llvm.amdgcn.update.dpp.p3(ptr addrspace(3) poison, ptr addrspace(3) %in, i32 337, i32 1, i32 1, i1 0) + store ptr addrspace(3) %tmp0, ptr addrspace(1) %out + ret void +} + +; GCN-LABEL: update_dpp_p0_poison: +; +; GFX90A-DAG: v_mov_b32_dpp v0, v0 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; GFX90A-DAG: v_mov_b32_dpp v1, v1 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GFX942-DAG: v_mov_b64_dpp v[0:1], v[0:1] row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GCN: global_store_dwordx2 v[2:3], v[0:1], off + +define amdgpu_ps void @update_dpp_p0_poison(ptr %in, ptr addrspace(1) %out) { + %tmp0 = call ptr @llvm.amdgcn.update.dpp.p0(ptr poison, ptr %in, i32 337, i32 1, i32 1, i1 0) + store ptr %tmp0, ptr addrspace(1) %out + ret void +} + +; GCN-LABEL: update_dpp_i64_unsupported_dpp64_op: +; +; GCN-DAG: v_mov_b32_dpp v3, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 +; GCN-DAG: v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 +; +; GCN: global_store_dwordx2 v[4:5], v[2:3], off + +define amdgpu_ps void @update_dpp_i64_unsupported_dpp64_op(i64 %in, i64 %old, ptr addrspace(1) %out) { + %tmp0 = call i64 @llvm.amdgcn.update.dpp.i64(i64 %old, i64 %in, i32 1, i32 1, i32 1, i1 0) + store i64 %tmp0, ptr addrspace(1) %out + ret void +} + +; GCN-LABEL: update_dpp_i16: +; +; GCN: v_mov_b32_dpp v1, v0 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GCN: global_store_short v[2:3], v1, off + +define amdgpu_ps void @update_dpp_i16(i16 %in, i16 %old, ptr addrspace(1) %out) { + %tmp0 = call i16 @llvm.amdgcn.update.dpp.i16(i16 %old, i16 %in, i32 337, i32 1, i32 1, i1 0) + store i16 %tmp0, ptr addrspace(1) %out + ret void +} + +; GCN-LABEL: update_dpp_half: +; +; GCN-DAG: v_mov_b32_dpp v1, v0 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GCN: global_store_short v[2:3], v1, off + +define amdgpu_ps void @update_dpp_half(half %in, half %old, ptr addrspace(1) %out) { + %tmp0 = call half @llvm.amdgcn.update.dpp.f16(half %old, half %in, i32 337, i32 1, i32 1, i1 0) + store half %tmp0, ptr addrspace(1) %out + ret void +} + +; GCN-LABEL: update_dpp_bfloat: +; +; GCN-DAG: v_mov_b32_dpp v1, v0 row_newbcast:1 row_mask:0x1 bank_mask:0x1 +; +; GCN: global_store_short v[2:3], v1, off + +define amdgpu_ps void @update_dpp_bfloat(bfloat %in, bfloat %old, ptr addrspace(1) %out) { + %tmp0 = call bfloat @llvm.amdgcn.update.dpp.bf16(bfloat %old, bfloat %in, i32 337, i32 1, i32 1, i1 0) + store bfloat %tmp0, ptr addrspace(1) %out + ret void +} + +; GCN-LABEL: update_dpp_v2p0_poison: +; +; GFX90A-DAG: v_mov_b32_dpp v0, v0 row_newbcast:1 row_mask:0x1 bank_mask:0x2 +; GFX90A-DAG: v_mov_b32_dpp v1, v1 row_newbcast:1 row_mask:0x1 bank_mask:0x2 +; GFX90A-DAG: v_mov_b32_dpp v2, v2 row_newbcast:1 row_mask:0x1 bank_mask:0x2 +; GFX90A-DAG: v_mov_b32_dpp v3, v3 row_newbcast:1 row_mask:0x1 bank_mask:0x2 +; +; GFX942-DAG: v_mov_b64_dpp v[2:3], v[2:3] row_newbcast:1 row_mask:0x1 bank_mask:0x2 +; GFX942-DAG: v_mov_b64_dpp v[0:1], v[0:1] row_newbcast:1 row_mask:0x1 bank_mask:0x2 +; +; GCN: global_store_dwordx4 v[4:5], v[0:3], off + +define amdgpu_ps void @update_dpp_v2p0_poison(<2 x ptr> %in, ptr addrspace(1) %out) { + %tmp0 = call <2 x ptr> @llvm.amdgcn.update.dpp.v2p0(<2 x ptr> poison, <2 x ptr> %in, i32 337, i32 1, i32 2, i1 0) + store <2 x ptr> %tmp0, ptr addrspace(1) %out + ret void +} diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll index a9a710f9b6723c..a7424831ae5dbf 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll @@ -57,8 +57,10 @@ bb: ; GCN-LABEL: {{^}}update_dppi64_test: ; GCN: load_{{dwordx2|b64}} v[[[SRC_LO:[0-9]+]]:[[SRC_HI:[0-9]+]]] -; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} -; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-OPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-OPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-NOOPT: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-NOOPT: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} define amdgpu_kernel void @update_dppi64_test(ptr addrspace(1) %arg, i64 %in1, i64 %in2) { %id = tail call i32 @llvm.amdgcn.workitem.id.x() %gep = getelementptr inbounds i64, ptr addrspace(1) %arg, i32 %id @@ -70,8 +72,10 @@ define amdgpu_kernel void @update_dppi64_test(ptr addrspace(1) %arg, i64 %in1, i ; GCN-LABEL: {{^}}update_dppf64_test: ; GCN: load_{{dwordx2|b64}} v[[[SRC_LO:[0-9]+]]:[[SRC_HI:[0-9]+]]] -; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} -; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-OPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-OPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-NOOPT: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-NOOPT: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} define amdgpu_kernel void @update_dppf64_test(ptr addrspace(1) %arg, double %in1, double %in2) { %id = tail call i32 @llvm.amdgcn.workitem.id.x() %gep = getelementptr inbounds double, ptr addrspace(1) %arg, i32 %id @@ -83,8 +87,10 @@ define amdgpu_kernel void @update_dppf64_test(ptr addrspace(1) %arg, double %in1 ; GCN-LABEL: {{^}}update_dppv2i32_test: ; GCN: load_{{dwordx2|b64}} v[[[SRC_LO:[0-9]+]]:[[SRC_HI:[0-9]+]]] -; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} -; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-OPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-OPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-NOOPT: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-NOOPT: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} define amdgpu_kernel void @update_dppv2i32_test(ptr addrspace(1) %arg, <2 x i32> %in1, <2 x i32> %in2) { %id = tail call i32 @llvm.amdgcn.workitem.id.x() %gep = getelementptr inbounds <2 x i32>, ptr addrspace(1) %arg, i32 %id @@ -96,8 +102,10 @@ define amdgpu_kernel void @update_dppv2i32_test(ptr addrspace(1) %arg, <2 x i32> ; GCN-LABEL: {{^}}update_dppv2f32_test: ; GCN: load_{{dwordx2|b64}} v[[[SRC_LO:[0-9]+]]:[[SRC_HI:[0-9]+]]] -; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} -; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-OPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-OPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-NOOPT: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-NOOPT: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} define amdgpu_kernel void @update_dppv2f32_test(ptr addrspace(1) %arg, <2 x float> %in1, <2 x float> %in2) { %id = tail call i32 @llvm.amdgcn.workitem.id.x() %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %arg, i32 %id @@ -109,8 +117,10 @@ define amdgpu_kernel void @update_dppv2f32_test(ptr addrspace(1) %arg, <2 x floa ; GCN-LABEL: {{^}}update_dpp_p0_test: ; GCN: load_{{dwordx2|b64}} v[[[SRC_LO:[0-9]+]]:[[SRC_HI:[0-9]+]]] -; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} -; GCN-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-OPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_LO]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-OPT-DAG: v_mov_b32_dpp v{{[0-9]+}}, v[[SRC_HI]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-NOOPT: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} +; GCN-NOOPT: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1{{$}} define amdgpu_kernel void @update_dpp_p0_test(ptr addrspace(1) %arg, ptr %in1, ptr %in2) { %id = tail call i32 @llvm.amdgcn.workitem.id.x() %gep = getelementptr inbounds ptr, ptr addrspace(1) %arg, i32 %id