[comgr][hotswap] Lower M32 scale16 WMMA exactly

amd/comgr/src/comgr-hotswap-b0a0.cpp

@@ -4083,6 +4083,24 @@ struct FiniteControlFlowAudit {
   bool HasUnboundedIndirectEntries = false;
 };
 
+// Some B0-only vector encodings are intentionally absent from the A0 MC
+// decoder used by hotswap. The legacy VOP3 encoding has the exact six-bit
+// major 0x34 (Inst[31:26]): it cannot transfer control or write the scalar
+// MODE register. An undecoded instance therefore remains opaque to dataflow,
+// but it is not an object-wide indirect-entry source. Keep this whitelist on
+// the exact encoding class; every other undecoded encoding retains the
+// fail-closed behavior.
+static bool
+isProvablyNonControlFlowUndecodedVectorInst(const InternalDecodedInst &DI,
+                                            ArrayRef<uint8_t> Text) {
+  if (DI.DecodeSucceeded || DI.Offset > Text.size() ||
+      MinInstSize > Text.size() - DI.Offset)
+    return false;
+  uint32_t Word =
+      support::endian::read32le(Text.data() + static_cast<size_t>(DI.Offset));
+  return (Word & 0xfc000000u) == (0x34u << 26);
+}
+
 static FiniteControlFlowAudit auditFiniteIndirectControlFlow(
     ArrayRef<InternalDecodedInst> Decoded, const LLVMState &LS,
     uint64_t TextAddr, uint64_t TextSize,
@@ -4091,7 +4109,8 @@ static FiniteControlFlowAudit auditFiniteIndirectControlFlow(
     const ControlFlowScanIndex &Index,
     ArrayRef<FiniteSetPcTransfer> FiniteSetPcTransfers,
     ArrayRef<BoundedSetPcReturn> BoundedReturns,
-    ArrayRef<SymbolLessReturnRegion> SymbolLessRegions) {
+    ArrayRef<SymbolLessReturnRegion> SymbolLessRegions,
+    ArrayRef<uint8_t> Text) {
   FiniteControlFlowAudit Audit{BitVector(FiniteSetPcTransfers.size()), true};
   auto markUnboundedIndirectEntry = [&]() {
     Audit.Closed = false;
@@ -4301,7 +4320,9 @@ static FiniteControlFlowAudit auditFiniteIndirectControlFlow(
         markUnboundedIndirectEntry();
   }
   for (int I = Reachable.find_first(); I >= 0; I = Reachable.find_next(I))
-    if (!Decoded[static_cast<size_t>(I)].DecodeSucceeded)
+    if (!Decoded[static_cast<size_t>(I)].DecodeSucceeded &&
+        !isProvablyNonControlFlowUndecodedVectorInst(
+            Decoded[static_cast<size_t>(I)], Text))
       markUnboundedIndirectEntry();
   return Audit;
 }
@@ -5937,7 +5958,7 @@ std::optional<DirectControlFlowInfo> collectDirectBranchTargets(
     FiniteControlFlowAudit Audit = auditFiniteIndirectControlFlow(
         Decoded, LS, TextAddr, TextSize, FunctionRanges, DeclaredEntries,
         ExternalEntries, *Index, EnabledSetPcTransfers, AllBoundedReturns,
-        SymbolLessRegions);
+        SymbolLessRegions, Text);
     if (Audit.InvalidSetPcCandidates.any()) {
       for (size_t I = 0; I != EnabledSetPcTransfers.size(); ++I) {
         if (!Audit.InvalidSetPcCandidates.test(I))
@@ -5965,7 +5986,7 @@ std::optional<DirectControlFlowInfo> collectDirectBranchTargets(
       Audit = auditFiniteIndirectControlFlow(
           Decoded, LS, TextAddr, TextSize, FunctionRanges, DeclaredEntries,
           ExternalEntries, *Index, EnabledSetPcTransfers, AllBoundedReturns,
-          SymbolLessRegions);
+          SymbolLessRegions, Text);
     }
     if (!Audit.Closed && !EnabledSetPcTransfers.empty()) {
       for (const FiniteSetPcTransfer &Enabled : EnabledSetPcTransfers)

amd/comgr/src/comgr-hotswap-internal.h

@@ -1467,6 +1467,41 @@ struct PatchContext {
   llvm::StringMap<unsigned> KernelVgprGranuleCache;
 };
 
+/// One node in the all-path proof that an incoming physical VGPR value is
+/// killed before it can be observed. Opaque nodes and unsafe exits observe
+/// every still-live value conservatively. A safe terminal (s_endpgm or the
+/// patched site on a later loop iteration) observes none.
+struct ForwardVgprProofNode {
+  llvm::BitVector Uses;
+  llvm::BitVector FullDefs;
+  llvm::SmallVector<size_t, 2> Successors;
+  bool Opaque = false;
+  bool HasUnsafeExit = false;
+  bool SafeTerminal = false;
+
+  explicit ForwardVgprProofNode(unsigned MaxVgprs = 0)
+      : Uses(MaxVgprs), FullDefs(MaxVgprs) {}
+};
+
+/// Return physical VGPR values whose incoming contents are killed on every
+/// path before a use, opaque instruction, unsafe exit, or non-killing cycle.
+/// Malformed graph inputs fail closed with std::nullopt.
+std::optional<llvm::BitVector>
+computeForwardDeadVgprs(llvm::ArrayRef<ForwardVgprProofNode> Nodes,
+                        size_t EntryNode, unsigned MaxVgprs);
+
+/// True when [Base, Base + Width) is non-empty, within MaxVgprs, and does not
+/// cross one of gfx1250's 256-register physical VGPR banks.
+bool physicalVgprRangeFitsOneBank(unsigned Base, unsigned Width,
+                                  unsigned MaxVgprs);
+
+/// Return true when \p Reg or one of its aliases belongs to a physical vector
+/// register file. Physical-VGPR proofs use this after encoded-range recovery
+/// fails: a true result must invalidate the proof rather than silently treating
+/// the operand as scalar.
+bool isVectorRegisterOrAlias(llvm::MCRegister Reg,
+                             const llvm::MCRegisterInfo &MRI);
+
 enum class VgprMsbOperand : unsigned {
   Src0 = 0,
   Src1 = 2,
@@ -1490,6 +1525,13 @@ void ensureVgprMsbModes(PatchContext &Ctx);
 [[nodiscard]] std::optional<unsigned>
 getLocallyEstablishedVgprMsbMode(PatchContext &Ctx, size_t Idx);
 
+/// Apply one instruction's persistent VGPR-MSB transfer to an exact packed
+/// mode. Returns VgprMsbUnknown when the incoming state or the instruction's
+/// MODE effect is ambiguous; an exact setter can recover an exact mode.
+int16_t transferExactVgprMsbMode(int16_t Incoming,
+                                 const InternalDecodedInst &DI,
+                                 const LLVMState &LS);
+
 unsigned getVgprMsbBank(unsigned Mode, VgprMsbOperand Operand);
 void setVgprMsbBank(unsigned &Mode, VgprMsbOperand Operand, unsigned Bank);