diff --git a/docs/designs/vpto-soft-postupdate-design-zh.md b/docs/designs/vpto-soft-postupdate-design-zh.md index a9982d69e0..50d9c26487 100644 --- a/docs/designs/vpto-soft-postupdate-design-zh.md +++ b/docs/designs/vpto-soft-postupdate-design-zh.md @@ -36,27 +36,20 @@ bisheng 内部将候选指令分为两个处理分支: | 分支 | PTOAS Op | 非 Post intrinsic | Post intrinsic | |------|----------|-------------------|----------------| | Auto | `pto.vlds` | `llvm.hivm.vldsx1.v{N}{ty}` | `llvm.hivm.vldsx1.post.v{N}{ty}` | +| Auto | `pto.vldsx2` | `llvm.hivm.vldsx2.v{N}{llvmTy}` | `llvm.hivm.vldsx2.post.v{N}{llvmTy}` | +| Auto | `pto.plds` | `llvm.hivm.plds.b8` | `llvm.hivm.plds.post.b8` | +| Auto | `pto.pldi` | `llvm.hivm.pldi.b8` | `llvm.hivm.pldi.post.b8` | | Auto | `pto.vsts` | `llvm.hivm.vstsx1.v{N}{ty}` | `llvm.hivm.vstsx1.post.v{N}{ty}` | -| Auto | `pto.vsstb` | `llvm.hivm.vsstb` | `llvm.hivm.vsstb.post` | +| Auto | `pto.vsstb` | `llvm.hivm.vsstb.v{N}{llvmTy}` | `llvm.hivm.vsstb.post.v{N}{llvmTy}` | +| Auto | `pto.psts` | `llvm.hivm.psts.b8` | `llvm.hivm.psts.post.b8` | +| Auto | `pto.psti` | `llvm.hivm.psti.b8` | `llvm.hivm.psti.post.b8` | +| Auto | `pto.sprsts` | `llvm.hivm.sprsts` | `llvm.hivm.sprsts.post` | +| Auto | `pto.sprsti` | `llvm.hivm.sprsti` | `llvm.hivm.sprsti.post` | +| Auto | `pto.vstas` | `llvm.hivm.vstas` | `llvm.hivm.vstas.post` | +| Auto | `pto.vsldb` | `llvm.hivm.vsldb.v{N}{llvmTy}` | `llvm.hivm.vsldb.post.v{N}{llvmTy}` | LLVM lowering 时根据 op 是否有 `updated_base` 结果来选择生成 post 或非 post intrinsic。 -### 2.2 PTOAS 有 Op 但尚未实现 Post variant 的指令 - -这些指令结构上可支持 `updated_base`,但当前 ODS 定义中没有该可选返回值。 - -| 分支 | PTOAS Op | 当前 Intrinsic | 备注 | -|------|----------|---------------|------| -| Auto | `pto.vldsx2` | `llvm.hivm.vldsx2.v{N}{ty}` | `vlds` 的双向量变体 | -| Auto | `pto.vsldb` | `llvm.hivm.vsldb` | `vsstb` 的加载对称体(块步长加载) | -| Auto | `pto.plds` | `llvm.hivm.plds.b8` | predicate mask 加载(strided) | -| Auto | `pto.pldi` | `llvm.hivm.pldi.b8` | predicate mask 加载(interleaved) | -| Auto | `pto.psts` | `llvm.hivm.psts.b8` | predicate mask 存储(strided) | -| Auto | `pto.psti` | `llvm.hivm.psti.b8` | predicate mask 存储(interleaved) | -| Auto | `pto.vstas` | `llvm.hivm.vstas` | align 存储(带 offset) | -| Auto | `pto.sprsts` | `llvm.hivm.sprsts` | 标量 predicate 寄存器存储(strided) | -| Auto | `pto.sprsti` | `llvm.hivm.sprsti` | 标量 predicate 寄存器存储(interleaved) | - ### 2.3 Stateful Post-Update 指令(Mechanism B:align 状态穿针) 这些指令通过显式的 align 寄存器跟踪状态,**始终返回**更新后的 align,没有非 Post 形式。 @@ -134,7 +127,7 @@ pass 的驱动分为两个阶段。两个阶段都通过 `PostUpdateTable`(sta 循环路径对 base 和 strideOperand **各自独立**分析:每个操作数先试 **累加器分析**(优先),未命中再退到 **delta 分析**(兜底),两者的结果最后按 4.2.1 的公式合并为 `stride_new`。前者处理该操作数已通过 `iter_args` 显式累加的场景(stride 可以是任意已计算的值),后者处理从 IV 全新计算、无累加器的场景(stride 须为循环不变量)。因此同一条指令的 base 走累加器、strideOperand 走 delta 是允许的组合。 -两类指令(vlds/vsts 与 vsstb/vsldb)的分析和改写通过统一的地址描述符抽象,共享同一套分析流程。 +各类内存指令的分析和改写通过统一的地址描述符抽象,共享同一套分析流程。 无论走哪条路径,分析都只产出**符号表达式**,不触碰 IR;确认候选可行后才在单一插入点物化(见 4.2.2)。 @@ -143,7 +136,7 @@ pass 的驱动分为两个阶段。两个阶段都通过 `PostUpdateTable`(sta 每个候选 op 由 `PostUpdateOpInfo` 描述:base 与 strideOperand 的操作数下标,以及 strideOperand 的**单位**。 ``` -enum class StrideUnit { Element, Block, Byte }; +enum class StrideUnit { Element, Block, Alignment, Byte }; struct PostUpdateOpInfo { int baseOperandIdx; int strideOperandIdx; @@ -160,10 +153,17 @@ struct PostUpdateOpInfo { | 指令 | base | strideOperand | strideUnit | unitBytes | 有效地址 | |------|------|---------------|-----------|-----------|---------| | vlds/vsts | source/destination | offset (Index) | Element | elemBytes | base + offset | +| vldsx2(Step 4) | source | offset (Index) | Element | elemBytes | base + offset | | vsstb/vsldb | destination/source | repeat_stride (I16) | Block | 32 | dest + (32/elemBytes)·repeat_stride | -| sprsts/sprsti(Step 4) | destination | offset (I32) | Byte | 1 | dest + offset/elemBytes | +| plds/psts(Step 4) | source/destination | offset (Index) | Byte | 1 | base + offset/elemBytes | +| pldi(Step 4) | source | offset (Index) | Alignment | NORM: VL/8;US: VL/16;DS: min(32, VL/4) | base + (unitBytes/elemBytes)·offset | +| psti(Step 4) | destination | offset (Index) | Alignment | NORM: VL/8;PK: VL/16 | base + (unitBytes/elemBytes)·offset | +| sprsts(Step 4) | destination | offset (I32) | Byte | 1 | dest + offset/elemBytes | +| sprsti(Step 4) | destination | offset (I32) | Alignment | AR: 4 | dest + (4/elemBytes)·offset | +| vstas(Step 4) | destination | offset (I32) | Element | elemBytes | dest + offset | -> 新增指令时判断单位的方法:看它在 `VPTOLLVMEmitter.cpp` 的 lowering pattern 里,strideOperand 是否过 `convertElementOffsetToBytes`(→ Element),是否经 `packBlockRepeatStride` 透传控制字(→ Block),还是原样透传且 ISA 文档标注为字节(→ Byte)。 +> intrinsic 参数原样透传不能单独证明硬件地址单位;Step 4 的 immediate/scalar 差异以 CANN 9.1 SIM 的实际更新地址为准。 +> 上表中的 VL 以字节计;A5 的 VL 为 256 bytes,因此 pldi 的 NORM/US/DS 分别为 32/16/32 bytes,psti 的 NORM/PK 分别为 32/16 bytes。其他目标必须提供自己的查询结果,否则该候选不改写。 分析和改写的核心公式统一以字节表达: @@ -313,9 +313,9 @@ delta 分析同样是纯符号的:表中每一行返回 `StrideExpr`,结果 改写步骤对所有指令统一: -1. **物化 stride。** 叶子全部循环不变时发射到循环外,否则发射到候选 op 之前。常量按 `(值, 类型)` 在同一循环内复用同一个 SSA 值——4.2.7 的分组按 `(base, stride_new)` 的 **Value 同一性** 判定,重复创建等值常量会把本可共享 `iter_arg` 的 op 拆成多组。 +1. **物化 stride。** 叶子全部循环不变时发射到循环外,否则发射到候选 op 之前。常量按 `(值, 类型)` 在同一循环内复用同一个 SSA 值——4.2.7 的分组按 stride 的 **Value 同一性** 判定,重复创建等值常量会把本可共享 `iter_arg` 的 op 拆成多组。 -2. 计算初始指针 `init_ptr = pto.addptr(base_0, (unitBytes/elemBytes)·strideOperand_0)`(见 4.2.1;若偏移为零则直接用 `base_0`)。 +2. 计算初始指针 `init_ptr = pto.addptr(base_0, (unitBytes/elemBytes)·strideOperand_0)`(见 4.2.1;若偏移为零则直接用 `base_0`)。传给 `pto.addptr` 前将最终偏移规范为 `index`;Block 单位保持无符号扩展,其他单位使用有符号扩展,避免丢失 `sprsti` 负立即数的语义。 3. 新增指针类型的 `iter_arg`,初始值为 `init_ptr`。 4. 创建 Post-Update op:将 `strideOperand` 替换为 `stride_new`,base 替换为 iter_arg 的 block argument。其余操作数(block_stride、mask、dist 等)不变。 5. 将 `updated_base` 通过 `scf.yield` 传出。 @@ -345,9 +345,9 @@ Post-Update 模式下 `repeat_stride` 从地址偏移变为指针前进量,因 #### 4.2.7 同一循环中的多个 Op -两个 op 能共享同一个 `iter_arg`,当且仅当它们走**同一条地址序列**——起点 `init_ptr`(由 `base_0` 与 `strideOperand_0` 决定,见 4.2.1)相同,且步长 `stride_new` 相同。 +两个 op 能共享同一个 `iter_arg`,当且仅当它们走**同一条地址序列**——起点 `init_ptr`(由 `base_0`、`strideOperand_0` 和 `unitBytes` 决定,见 4.2.1)相同,且以字节计的步长相同。 -理想的分组键是 `(init_ptr, stride_new)`。但 `init_ptr` 不适合直接入键:分组按 **Value 同一性** 比较,而 `computeInitialPtr` 可能为每个候选各自物化一个 `pto.addptr`,起点数值相同也未必是同一个 SSA 值。因此改用决定 `init_ptr` 的**原始操作数**:分组键取 `(base, strideOperand, stride_new)`。操作数相同必然起点相同,这是一个充分条件——可能把本可合并的组拆开,但绝不会合并本应分开的组。 +理想的分组键是 `(init_ptr, byte_stride)`。但 `init_ptr` 不适合直接入键:分组按 **Value 同一性** 比较,而 `computeInitialPtr` 可能为每个候选各自物化一个 `pto.addptr`,起点数值相同也未必是同一个 SSA 值。因此改用决定地址序列的原始量:分组键取 `(base, strideOperand, stride_new, unitBytes)`。加入 `unitBytes` 可防止 f32 上数值相同的 Element 与 Byte stride 被误合并;该键可能把本可合并的组拆开,但不会合并字节递推不同的组。 同组的 op 共享一个 `iter_arg`,所有 op 使用同一个 pre-update 指针(block argument),不链式传递 `updated_base`。原因:同一迭代内同组 op 访问相同地址,链式传递会使后续 op 的地址偏移一个 stride。每组只需 yield 一个 `updated_base`。 @@ -539,50 +539,3 @@ def VPTOSoftPostUpdate : Pass<"vpto-soft-postupdate", "ModuleOp"> { "arith::ArithDialect"]; } ``` - -## 6. 实施计划 - -### ~~Step 1:pass 框架与循环 delta 分析~~(已完成) - -1. ~~搭建 pass 框架:PostUpdateSet、遍历逻辑、pipeline 集成、CLI 开关。~~ -2. ~~实现 delta 递归分析(4.2.4)。~~ -3. ~~实现 delta 路径的合法性检查和改写(新增 iter_arg)。~~ -4. ~~支持同一循环中多个 op。~~ -5. ~~支持 `pto.vlds`、`pto.vsts`、`pto.vsstb`。~~ -6. ~~添加 delta 路径的 lit 测试和反向测试。~~ - -### ~~Step 2:循环累加器分析~~(已完成) - -7. ~~实现 `getIterArgIncrement` helper。~~ -8. ~~实现累加器分析(4.2.3):base/offset 统一检测与 stride 合并。~~ -9. ~~实现累加器路径的改写(4.2.6)。~~ -10. ~~在遍历逻辑中将累加器分析置于 delta 分析之前。~~ -11. ~~添加累加器路径的 lit 测试(含非常量 stride、base 和 offset 都是 iter_arg 等场景)。~~ -12. ~~实现 `StrideExpr` 符号表达式与常量折叠(4.2.2),使 `decomposeLinear` / `getIterArgIncrement` / `computeDelta` 成为纯函数并按 `Value` 缓存。~~ -13. ~~实现 `getIterArgIncrement` 的三态返回(`NotIterArg` / `Failed` / `Ok`)。~~ -14. ~~实现物化阶段:类型一致性检查、叶子可用性检查(必要时克隆 pure 定义链)、常量按 `(值, 类型)` 复用。~~ -15. ~~补充回归测试:增量定义在消费者之后、多增量组合、weight=32 非常量增量的负向用例。~~ - -### ~~Step 3:顺序路径~~(已完成) - -16. ~~抽取循环与顺序路径共享的 helper:精确 elemBytes/unitBytes 换算、`combineStride`、初始指针计算和 post-update op 泛型构造。~~ -17. ~~实现 base 归一化:沿 `pto.addptr` 得到 `(rootBase, baseOffset)`;复用 `StrideExpr` 构造规则,并增加仿射规范化、相等比较和按系数精确缩放,使两条路径都能证明 `8*k` 一类非常量表达式的单位整除性;顺序路径仅在保留 cast 后能按目标 stride 类型精确物化时改写。~~ -18. ~~调整驱动为两阶段:先内到外完成全部循环路径,再重新收集所有 block(包括 `scf.for` body)和剩余非 Post-Update op。~~ -19. ~~实现有序分桶与线性 `SequentialRun` 检测:按 `(op 类型, rootBase)` 分桶,允许不同桶物理交错;接受的 `SequentialRun` 互不重叠,被拒 `SequentialRun` 从 `end - 1` 保留一个尾候选重试。~~ -20. ~~复用循环路径的类型、精确换算、可用性和 pure 克隆检查,在整条 `SequentialRun` 通过后物化 step 并链式改写。~~ -21. ~~更新 `Passes.td` 中 pass 描述,使其同时覆盖循环递推和 block 内顺序访问。~~ -22. ~~添加 `test/lit/vpto` 回归测试:固定 base 常量序列、变化 base 与 offset 抵消、非常量仿射 step、Block 类 `8*k` 单位精确缩放及宽动态结果无法安全写入 i16 的负向用例、for-body 循环路径未命中后转顺序路径、不同桶交错、多个最大 `SequentialRun`、公差破坏、零步长及无法精确换算的负向用例。~~ -23. ~~将尾指令改为使用当前指针和同类型零 stride 的 normal 形式,避免产生无人使用的最终 `updated_base`。~~ -24. ~~增加独立收益性检查:接受能够删除足量动态多层 `pto.addptr` 的常量-step run,以及任意合法长度的 direct symbolic-leaf run;规则统一适用于当前支持的全部 op。~~ - -### Step 4:扩展指令覆盖 - -25. 为 2.2 中的指令(`vldsx2`、`vsldb`、`plds`、`pldi` 等)添加 ODS `updated_base` 定义。 -26. 扩展 `PostUpdateTable`,为每条新指令按 4.2.1 的方法确定 `StrideUnit`(Element / Block / Byte)。 -27. 补充对应的 LLVM lowering(post intrinsic callee)和 lit 测试。 - -### Step 5:验证与开启 - -28. 端到端验证:用 ptoas 编译,与 bisheng Post-Update 输出对比已知 kernel。 -29. NPU 验证:在硬件上运行 Post-Update kernel(通过现有 `test/vpto/cases/micro-op/vector-load-store/` 框架)。 -30. 将默认值切换为开启。 diff --git a/docs/isa/micro-isa/03-vector-load-store.md b/docs/isa/micro-isa/03-vector-load-store.md index 2e66f05c03..2cd35bc8e6 100644 --- a/docs/isa/micro-isa/03-vector-load-store.md +++ b/docs/isa/micro-isa/03-vector-load-store.md @@ -186,13 +186,17 @@ deinterleave forms. ### `pto.vldus` - **syntax:** `%result, %align_out = pto.vldus %source, %align : !pto.ptr, !pto.align -> !pto.vreg, !pto.align` +- **post-update syntax:** `%result, %align_out, %base_out = pto.vldus %source, %align, %increment : !pto.ptr, !pto.align, index -> !pto.vreg, !pto.align, !pto.ptr` - **semantics:** Unaligned load using primed align state. - **inputs:** `%source` is the current UB address and `%align` is the incoming load - alignment state primed by `pto.vldas` or a prior `pto.vldus`. + alignment state primed by `pto.vldas` or a prior `pto.vldus`. In the + post-update form, `%increment` is the number of `T` elements by which the + base advances after this access. - **outputs:** `%result` is the assembled vector value and `%align_out` is the updated - alignment state. + alignment state. The post-update form additionally returns `%base_out`, + equivalent to `%source` advanced by `%increment` elements. - **constraints and limitations:** A matching `pto.vldas` MUST appear before the first dependent `pto.vldus` stream in the same vector loop. The installed no-post A5 interface keeps a @@ -201,13 +205,16 @@ deinterleave forms. value and only exposes the updated align carrier. Reusing the original `%source` starts a new explicit access point; if the caller wants another no-post access, it should compute the next source pointer explicitly and pair - it with the required align setup. + it with the required align setup. The align carrier already has its own SSA + update chain in both forms; enabling base post-update does not change that + chain's semantics. - **Latency:** **9** cycles. **Unaligned load pattern:** ```mlir %align = pto.vldas %ub : !pto.ptr -> !pto.align %vec, %align2 = pto.vldus %ub, %align : !pto.ptr, !pto.align -> !pto.vreg<64xf32>, !pto.align +%vec2, %align3, %next = pto.vldus %ub, %align2, %c64 : !pto.ptr, !pto.align, index -> !pto.vreg<64xf32>, !pto.align, !pto.ptr ``` --- @@ -230,13 +237,14 @@ deinterleave forms. ### `pto.vldsx2` -- **syntax:** `%low, %high = pto.vldsx2 %source[%offset], "DIST" : !pto.ptr, index -> !pto.vreg, !pto.vreg` +- **syntax:** `%low, %high [, %updated_base] = pto.vldsx2 %source[%offset], "DIST" : !pto.ptr, index -> !pto.vreg, !pto.vreg [, !pto.ptr]` - **semantics:** Dual load with deinterleave (AoS → SoA conversion). - **inputs:** `%source` is the UB base pointer, `%offset` is the displacement, and `DIST` selects a dual-load/deinterleave layout. - **outputs:** - `%low` and `%high` are the two destination vectors. + `%low` and `%high` are the two destination vectors. If requested, + `%updated_base` is the source pointer advanced by `%offset`. - **constraints and limitations:** This family is only legal for interleave/deinterleave style distributions. The two outputs form an ordered pair, and that pairing MUST be preserved. @@ -269,14 +277,15 @@ for (int i = 0; i < 64; i++) { ### `pto.vsldb` -- **syntax:** `%result = pto.vsldb %source, %block_stride, %repeat_stride, %mask : !pto.ptr, i16, i16, !pto.mask -> !pto.vreg` +- **syntax:** `%result [, %updated_base] = pto.vsldb %source, %block_stride, %repeat_stride, %mask : !pto.ptr, i16, i16, !pto.mask -> !pto.vreg [, !pto.ptr]` - **semantics:** Block-strided load for 2D tile access. - **inputs:** `%source` is the UB base pointer. `%block_stride` and `%repeat_stride` are the two 16-bit fields of the hardware control word, and `%mask` controls which blocks participate. - **outputs:** - `%result` is the loaded vector. + `%result` is the loaded vector. If requested, `%updated_base` is the source + pointer advanced by `%repeat_stride` 32-byte blocks. - **constraints and limitations:** PTO surface does not expose the packed control word directly. If a block is masked off, the corresponding destination block is zeroed and MUST NOT raise @@ -560,42 +569,52 @@ for (int i = 0; i < num_requests; i++) ### `pto.sprsti` -- **syntax:** `pto.sprsti "AR", %dest[%offset] : !pto.ptr, i32` +- **syntax:** + - normal: `pto.sprsti "AR", %dest[%offset] : !pto.ptr, i32` + - post-update: `%next = pto.sprsti "AR", %dest[%offset] : !pto.ptr, i32 -> !pto.ptr` - **semantics:** Store SPR AR to UB using a signed 8-bit immediate offset. - **inputs:** - `%dest` is the UB base pointer and `%offset` is the immediate offset in units - of the SPR data width. + `%dest` is the UB base pointer and `%offset` counts 4-byte words. +- **outputs:** + The normal form returns no SSA value. The post-update form returns `%next`, + which advances `%dest` by `4 * %offset` bytes. - **constraints and limitations:** Only `"AR"` is supported. `%dest` must be a `ui32` or signless `i32` UB - pointer. `%offset` must be a constant signed 8-bit `i32`. The current VPTO - surface models only the no-post-update form, so no updated base pointer is - returned. + pointer. `%offset` must be a constant signed 8-bit `i32`. --- ### `pto.sprsts` -- **syntax:** `pto.sprsts "AR", %dest[%offset] : !pto.ptr, i32` +- **syntax:** + - normal: `pto.sprsts "AR", %dest[%offset] : !pto.ptr, i32` + - post-update: `%next = pto.sprsts "AR", %dest[%offset] : !pto.ptr, i32 -> !pto.ptr` - **semantics:** Store SPR AR to UB using a scalar-register offset. - **inputs:** `%dest` is the UB base pointer and `%offset` is the scalar offset in bytes. +- **outputs:** + The normal form returns no SSA value. The post-update form returns `%next`, + which advances `%dest` by `%offset` bytes. - **constraints and limitations:** Only `"AR"` is supported. `%dest` must be a `ui32` or signless `i32` UB - pointer. The current VPTO surface models only the no-post-update form, so no - updated base pointer is returned. + pointer. --- ## Alignment State Stores ### `pto.vstas` -- **syntax:** `pto.vstas %value, %dest, %offset : !pto.align, !pto.ptr, i32` +- **syntax:** + - normal: `pto.vstas %value, %dest, %offset : !pto.align, !pto.ptr, i32` + - post-update: `%next = pto.vstas %value, %dest, %offset : !pto.align, !pto.ptr, i32 -> !pto.ptr` - **semantics:** Scalar-register-offset form of alignment-state flush. - **inputs:** `%value` is the pending store-alignment state, `%dest` is the UB base - pointer, and `%offset` is the scalar-register style displacement. + pointer, and `%offset` is the displacement in destination elements. - **outputs:** - This op writes buffered tail bytes to UB and returns no SSA value. + This op writes buffered tail bytes to UB. The normal form returns no SSA + value. The post-update form returns `%next`, which advances `%dest` by + `%offset` destination elements. - **constraints and limitations:** This family flushes pending store-alignment state using an explicit scalar offset and keeps the scalar-offset form explicit. The incoming `%value` @@ -649,13 +668,15 @@ These ops make reference-updated state explicit as SSA results. ### `pto.vstus` - **syntax:** `%align_out = pto.vstus %align_in, %offset, %value, %base : !pto.align, i32, !pto.vreg, !pto.ptr -> !pto.align` -- **semantics:** No-post unaligned store with scalar offset. +- **post-update syntax:** `%align_out, %base_out = pto.vstus %align_in, %offset, %value, %base : !pto.align, i32, !pto.vreg, !pto.ptr -> !pto.align, !pto.ptr` +- **semantics:** Unaligned store with scalar stream advance and optional base post-update. - **inputs:** - `%align_in` is the incoming store-alignment state, `%offset` is the scalar - displacement, `%value` is the vector being stored, and `%base` is the UB base - pointer. + `%align_in` is the incoming store-alignment state, `%offset` is the number of + `T` elements by which the store stream advances, `%value` is the vector being + stored, and `%base` is the UB base pointer. - **outputs:** - `%align_out` is the updated buffered-tail state. + `%align_out` is the updated buffered-tail state. In the post-update form, + `%base_out` is `%base` advanced by `%offset` elements. - **constraints and limitations:** This is the scalar-offset stateful form of the unaligned store family. The scalar offset width MUST match the selected form, and a later flush op is @@ -664,8 +685,10 @@ These ops make reference-updated state explicit as SSA results. `%base + %offset`". Instead, `%offset` describes how far the store stream advances at this step, and `%align_out` carries any residual tail that could not be committed yet. The no-post surface does not expose an updated base - pointer. A later flush op must therefore use an explicit destination/offset - pair that identifies the same logical flush point as this `pto.vstus`. + pointer, while the post-update form returns that pointer directly. The align + carrier remains an independent SSA state chain in either form. A later flush + op must use a destination/offset pair that identifies the same logical flush + point as this `pto.vstus` stream. - **Latency:** **9** cycles. --- diff --git a/docs/isa/micro-isa/04-predicate-load-store.md b/docs/isa/micro-isa/04-predicate-load-store.md index 9c3bed11df..175f2ea586 100644 --- a/docs/isa/micro-isa/04-predicate-load-store.md +++ b/docs/isa/micro-isa/04-predicate-load-store.md @@ -10,8 +10,18 @@ examples below use `b32` when the loaded/stored mask is used with `f32` vector compares or selects. The predicate load/store ops documented on this page always use explicit -`base[offset]` addressing. The immediate forms (`pldi`, `psti`) and dynamic -forms (`plds`, `psts`) differ only in how `%offset` is supplied. +`base[offset]` addressing. Scalar forms (`plds`, `psts`) express `%offset` in +bytes. For immediate forms (`pldi`, `psti`), one `%offset` unit is an alignment +unit selected by `DIST`. Here `VL` is the target vector length in bytes; on A5, +`VL = 256 bytes`: + +| op | `DIST` | one immediate offset unit | A5 value | +|----|--------|---------------------------|----------| +| `pldi` | `NORM` | `VL / 8` | 32 bytes | +| `pldi` | `US` | `VL / 16` | 16 bytes | +| `pldi` | `DS` | `min(32 bytes, VL / 4)` | 32 bytes | +| `psti` | `NORM` | `VL / 8` | 32 bytes | +| `psti` | `PK` | `VL / 16` | 16 bytes | --- @@ -19,7 +29,7 @@ forms (`plds`, `psts`) differ only in how `%offset` is supplied. ### `pto.plds` -- **syntax:** `%result = pto.plds %source[%offset], "DIST" : !pto.ptr, index -> !pto.mask` +- **syntax:** `%result [, %updated_base] = pto.plds %source[%offset], "DIST" : !pto.ptr, index -> !pto.mask [, !pto.ptr]` - **semantics:** Load predicate register with runtime offset. This is the dynamic-offset form of `pto.pldi`: the predicate payload interpretation is the same, but `%offset` is supplied as an SSA `index` instead of a constant @@ -35,6 +45,7 @@ The loaded payload is a packed predicate image in UB. Consumer ops interpret the resulting `!pto.mask` according to the mask granularity `G`. `pto.plds` only models the explicit `base[offset]` form. +If requested, `%updated_base` is `%source` advanced by `%offset` bytes. **Example:** ```mlir @@ -45,8 +56,9 @@ models the explicit `base[offset]` form. ### `pto.pldi` -- **syntax:** `%result = pto.pldi %source[%offset], "DIST" : !pto.ptr, index -> !pto.mask` +- **syntax:** `%result [, %updated_base] = pto.pldi %source[%offset], "DIST" : !pto.ptr, index -> !pto.mask [, !pto.ptr]` - **offset:** must be a constant `index` immediate in PTO surface form. +- **offset unit:** the `DIST`-selected alignment unit from the table above. - **semantics:** Load predicate register with immediate offset. - **DIST:** mandatory string token, one of `NORM`, `US`, `DS`. - `NORM`: load a normal packed predicate payload of size `VL/8`. @@ -57,6 +69,16 @@ models the explicit `base[offset]` form. Like `pto.plds`, this op reads a packed predicate payload from UB and materializes it as `!pto.mask`. +If requested, `%updated_base` is `%source` advanced by the immediate +`alignment_unit(DIST) * %offset` bytes. + +**Example:** on A5, `US` selects a 16-byte alignment unit, so `%c1` accesses +`%source + 16 bytes` and returns that address as `%next`: + +```mlir +%mask, %next = pto.pldi %source[%c1], "US" + : !pto.ptr, index -> !pto.mask, !pto.ptr +``` --- @@ -64,7 +86,7 @@ materializes it as `!pto.mask`. ### `pto.psts` -- **syntax:** `pto.psts %value, %dest[%offset], "DIST" : !pto.mask, !pto.ptr, index` +- **syntax:** `[%updated_base =] pto.psts %value, %dest[%offset], "DIST" : !pto.mask, !pto.ptr, index [-> !pto.ptr]` - **semantics:** Store predicate register with runtime offset. This is the dynamic-offset form of `pto.psti`: the predicate payload interpretation is the same, but `%offset` is supplied as an SSA `index` instead of a constant @@ -77,6 +99,7 @@ materializes it as `!pto.mask`. `pto.psts` stores the packed predicate payload represented by `!pto.mask`. It only models the explicit `base[offset]` form. +If requested, `%updated_base` is `%dest` advanced by `%offset` bytes. **Example:** ```mlir @@ -87,8 +110,9 @@ pto.psts %mask, %ub[%c0], "NORM" : !pto.mask, !pto.ptr, index ### `pto.psti` -- **syntax:** `pto.psti %value, %dest[%offset], "DIST" : !pto.mask, !pto.ptr, index` +- **syntax:** `[%updated_base =] pto.psti %value, %dest[%offset], "DIST" : !pto.mask, !pto.ptr, index [-> !pto.ptr]` - **offset:** must be a constant `index` immediate in PTO surface form. +- **offset unit:** the `DIST`-selected alignment unit from the table above. - **semantics:** Store predicate register with immediate offset. - **DIST:** mandatory string token, one of `NORM`, `PK`. - `NORM`: store the packed predicate payload into a normal destination space @@ -97,8 +121,16 @@ pto.psts %mask, %ub[%c0], "NORM" : !pto.mask, !pto.ptr, index `VL/16`, keeping one bit out of every two bits. `pto.psti` and `pto.psts` store the packed predicate payload represented by -`!pto.mask`. The surface distinction is only immediate-offset versus -dynamic-offset. +`!pto.mask`. +If requested, `%updated_base` is `%dest` advanced by the immediate `%offset` +multiplied by `alignment_unit(DIST)` bytes. + +**Example:** on A5, `PK` selects a 16-byte alignment unit: + +```mlir +%next = pto.psti %mask, %dest[%c1], "PK" + : !pto.mask, !pto.ptr, index -> !pto.ptr +``` --- diff --git a/docs/vpto-spec.md b/docs/vpto-spec.md index 4c13bc72ba..3951fd4a3f 100644 --- a/docs/vpto-spec.md +++ b/docs/vpto-spec.md @@ -1162,12 +1162,23 @@ dst[i] = mask[i] ? op(src0[i], src1[i]) : 0 // ZEROING mode ```mlir %align = pto.vldas %ub : !pto.ptr -> !pto.align %vec, %align_out = pto.vldus %ub, %align : !pto.ptr, !pto.align -> !pto.vreg<64xf32>, !pto.align +%vec2, %align_out2, %next_ub = pto.vldus %ub, %align_out, %increment + : !pto.ptr, !pto.align, index + -> !pto.vreg<64xf32>, !pto.align, !pto.ptr %store_align = pto.init_align : !pto.align %next_align = pto.vstus %store_align, %offset, %vec, %ub : !pto.align, i32, !pto.vreg<64xf32>, !pto.ptr -> !pto.align +%next_align2, %next_store_ub = pto.vstus %next_align, %offset, %vec2, %ub + : !pto.align, i32, !pto.vreg<64xf32>, !pto.ptr + -> !pto.align, !pto.ptr ``` +In the post-update forms, the base result advances by the load increment or +store offset in pointer-element units. The `!pto.align` result remains the +existing independent alignment-state update; base post-update does not alter +its meaning. + --- ## Part II: Notation Convention @@ -1214,6 +1225,7 @@ pto.vsts %value, %destination[%offset] {dist = "DIST"} : !pto.vreg, !pto.pt ```mlir %low, %high = pto.vldsx2 %source[%offset], "DIST" : !pto.ptr, index -> !pto.vreg, !pto.vreg +%low, %high, %updated_base = pto.vldsx2 %source[%offset], "DIST" : !pto.ptr, index -> !pto.vreg, !pto.vreg, !pto.ptr ``` **Dual Store (two inputs, one interleaved store):** diff --git a/include/PTO/IR/PTO.h b/include/PTO/IR/PTO.h index 2510d6fc07..6b3464d50b 100644 --- a/include/PTO/IR/PTO.h +++ b/include/PTO/IR/PTO.h @@ -175,6 +175,10 @@ class ScopedPTOParserTargetArch { PTOParserTargetArch previousArch; }; +/// Return the target-specific alignment size in bytes for a supported +/// load/store vector op. Unsupported operations, modes, and targets return +/// std::nullopt. +std::optional getLoadStoreVecAlignmentSize(Operation *op); /// Function attributes that mark an explicit PTO kernel entry. inline constexpr llvm::StringLiteral kPTOEntryAttrName = "pto.entry"; diff --git a/include/PTO/IR/VPTOOps.td b/include/PTO/IR/VPTOOps.td index 395807988f..c1a59b1cb9 100644 --- a/include/PTO/IR/VPTOOps.td +++ b/include/PTO/IR/VPTOOps.td @@ -1501,12 +1501,15 @@ def PTO_Vldsx2Op : PTO_VectorMicroOp<"vldsx2", [ Index:$offset, StrAttr:$dist ); - let results = (outs PTO_VectorType:$low, PTO_VectorType:$high); + let results = (outs PTO_VectorType:$low, + PTO_VectorType:$high, + Optional:$updated_base); let hasVerifier = 1; let assemblyFormat = [{ $source `[` $offset `]` `,` $dist attr-dict `:` type($source) `,` type($offset) `->` type($low) `,` type($high) + (`,` type($updated_base)^)? }]; } @@ -1559,12 +1562,13 @@ def PTO_SprstiOp : PTO_VectorMicroOp<"sprsti", [ I32:$offset ); - let results = (outs); + let results = (outs Optional:$updated_base); let hasVerifier = 1; let assemblyFormat = [{ $spr `,` $destination `[` $offset `]` attr-dict `:` type($destination) `,` type($offset) + (`->` type($updated_base)^)? }]; } @@ -1577,12 +1581,13 @@ def PTO_SprstsOp : PTO_VectorMicroOp<"sprsts", [ I32:$offset ); - let results = (outs); + let results = (outs Optional:$updated_base); let hasVerifier = 1; let assemblyFormat = [{ $spr `,` $destination `[` $offset `]` attr-dict `:` type($destination) `,` type($offset) + (`->` type($updated_base)^)? }]; } @@ -1591,18 +1596,22 @@ def PTO_VldusOp : PTO_VectorMicroOp<"vldus", [ ]> { let arguments = (ins PTO_BufferType:$source, - PTO_AlignTypeConstraint:$align + PTO_AlignTypeConstraint:$align, + Optional:$increment ); let results = (outs PTO_VectorType:$result, - PTO_AlignTypeConstraint:$updated_align + PTO_AlignTypeConstraint:$updated_align, + Optional:$updated_base ); let hasVerifier = 1; let assemblyFormat = [{ - $source `,` $align attr-dict `:` type($source) `,` type($align) `->` type($result) `,` type($updated_align) + $source `,` $align (`,` $increment^)? attr-dict `:` type($source) `,` type($align) + (`,` type($increment)^)? `->` type($result) `,` type($updated_align) + (`,` type($updated_base)^)? }]; } @@ -1894,12 +1903,14 @@ def PTO_PldsOp : PTO_VectorMicroOp<"plds", [ Index:$offset, StrAttr:$dist ); - let results = (outs PTO_MaskTypeConstraint:$result); + let results = (outs PTO_MaskTypeConstraint:$result, + Optional:$updated_base); let hasVerifier = 1; let assemblyFormat = [{ $source `[` $offset `]` `,` $dist attr-dict `:` type($source) `,` type($offset) `->` type($result) + (`,` type($updated_base)^)? }]; } @@ -1911,12 +1922,14 @@ def PTO_PldiOp : PTO_VectorMicroOp<"pldi", [ Index:$offset, StrAttr:$dist ); - let results = (outs PTO_MaskTypeConstraint:$result); + let results = (outs PTO_MaskTypeConstraint:$result, + Optional:$updated_base); let hasVerifier = 1; let assemblyFormat = [{ $source `[` $offset `]` `,` $dist attr-dict `:` type($source) `,` type($offset) `->` type($result) + (`,` type($updated_base)^)? }]; } @@ -1929,12 +1942,13 @@ def PTO_PstiOp : PTO_VectorMicroOp<"psti", [ Index:$offset, StrAttr:$dist ); - let results = (outs); + let results = (outs Optional:$updated_base); let hasVerifier = 1; let assemblyFormat = [{ $value `,` $destination `[` $offset `]` `,` $dist attr-dict `:` type($value) `,` type($destination) `,` type($offset) + (`->` type($updated_base)^)? }]; } @@ -2715,12 +2729,13 @@ def PTO_PstsOp : PTO_VectorMicroOp<"psts", [ StrAttr:$dist ); - let results = (outs); + let results = (outs Optional:$updated_base); let hasVerifier = 1; let assemblyFormat = [{ $value `,` $destination `[` $offset `]` `,` $dist attr-dict `:` type($value) `,` type($destination) `,` type($offset) + (`->` type($updated_base)^)? }]; } @@ -3464,12 +3479,14 @@ def PTO_VsldbOp : PTO_VectorMicroOp<"vsldb", [ I16:$repeat_stride, PTO_MaskTypeConstraint:$mask ); - let results = (outs PTO_VectorType:$result); + let results = (outs PTO_VectorType:$result, + Optional:$updated_base); let hasVerifier = 1; let assemblyFormat = [{ $source `,` $block_stride `,` $repeat_stride `,` $mask attr-dict `:` type($source) `,` type($block_stride) `,` type($repeat_stride) `,` type($mask) `->` type($result) + (`,` type($updated_base)^)? }]; } @@ -3501,12 +3518,13 @@ def PTO_VstasOp : PTO_VectorMicroOp<"vstas", [ PTO_BufferLikeType:$destination, I32:$offset ); - let results = (outs); + let results = (outs Optional:$updated_base); let hasVerifier = 1; let assemblyFormat = [{ $value `,` $destination `,` $offset attr-dict `:` type($value) `,` type($destination) `,` type($offset) + (`->` type($updated_base)^)? }]; } @@ -3557,12 +3575,14 @@ def PTO_VstusOp : PTO_VectorMicroOp<"vstus", [ PTO_VectorType:$value, PTO_BufferType:$base ); - let results = (outs PTO_AlignTypeConstraint:$align_out); + let results = (outs PTO_AlignTypeConstraint:$align_out, + Optional:$base_out); let hasVerifier = 1; let assemblyFormat = [{ $align_in `,` $offset `,` $value `,` $base attr-dict `:` type($align_in) `,` type($offset) `,` type($value) `,` type($base) `->` type($align_out) + (`,` type($base_out)^)? }]; } diff --git a/lib/PTO/IR/PTO.cpp b/lib/PTO/IR/PTO.cpp index 32acb853e8..82f034cc38 100644 --- a/lib/PTO/IR/PTO.cpp +++ b/lib/PTO/IR/PTO.cpp @@ -388,6 +388,92 @@ bool mlir::pto::isTargetArchA5(Operation *op) { return getTargetArch(op) == PTOArch::A5; } +constexpr int64_t kA5VectorLengthBytes = 256; + +enum class PredicateLoadDist { + Norm, + Us, + Ds, +}; + +enum class PredicateStoreDist { + Norm, + Pk, +}; + +struct PredicateLoadAlignmentRule { + PredicateLoadDist dist; + int64_t alignmentBytes; +}; + +struct PredicateStoreAlignmentRule { + PredicateStoreDist dist; + int64_t alignmentBytes; +}; + +constexpr PredicateLoadAlignmentRule kA5PredicateLoadAlignmentRules[] = { + {PredicateLoadDist::Norm, kA5VectorLengthBytes / 8}, + {PredicateLoadDist::Us, kA5VectorLengthBytes / 16}, + {PredicateLoadDist::Ds, std::min(32, kA5VectorLengthBytes / 4)}, +}; + +constexpr PredicateStoreAlignmentRule kA5PredicateStoreAlignmentRules[] = { + {PredicateStoreDist::Norm, kA5VectorLengthBytes / 8}, + {PredicateStoreDist::Pk, kA5VectorLengthBytes / 16}, +}; + +static std::optional +parsePredicateLoadDist(StringRef dist) { + if (dist == "NORM") + return PredicateLoadDist::Norm; + if (dist == "US") + return PredicateLoadDist::Us; + if (dist == "DS") + return PredicateLoadDist::Ds; + return std::nullopt; +} + +static std::optional +parsePredicateStoreDist(StringRef dist) { + if (dist == "NORM") + return PredicateStoreDist::Norm; + if (dist == "PK") + return PredicateStoreDist::Pk; + return std::nullopt; +} + +template +static std::optional findAlignmentSize(const Rule (&rules)[N], + Dist dist) { + auto rule = llvm::find_if( + rules, [&](const Rule &entry) { return entry.dist == dist; }); + if (rule == std::end(rules)) + return std::nullopt; + return rule->alignmentBytes; +} + +std::optional +mlir::pto::getLoadStoreVecAlignmentSize(Operation *op) { + if (!op || getTargetArch(op) != PTOArch::A5) + return std::nullopt; + + if (auto pldi = dyn_cast(op)) { + auto dist = parsePredicateLoadDist(pldi.getDist()); + return dist ? findAlignmentSize(kA5PredicateLoadAlignmentRules, *dist) + : std::nullopt; + } + if (auto psti = dyn_cast(op)) { + auto dist = parsePredicateStoreDist(psti.getDist()); + return dist ? findAlignmentSize(kA5PredicateStoreAlignmentRules, *dist) + : std::nullopt; + } + if (auto sprsti = dyn_cast(op)) { + if (sprsti.getSpr() == "AR") + return 4; + } + return std::nullopt; +} + static llvm::TypeSize getOneByteTypeSize() { return llvm::TypeSize::getFixed(8); } diff --git a/lib/PTO/IR/VPTO.cpp b/lib/PTO/IR/VPTO.cpp index a02eac07a8..dfe6a3458a 100644 --- a/lib/PTO/IR/VPTO.cpp +++ b/lib/PTO/IR/VPTO.cpp @@ -4926,9 +4926,14 @@ void SprstiOp::getEffects( } LogicalResult SprstiOp::verify() { - return verifySprStoreCommon(getOperation(), "pto.sprsti", getSpr(), - getDestination(), getOffset(), - /*requireImmediateOffset=*/true); + if (failed(verifySprStoreCommon(getOperation(), "pto.sprsti", getSpr(), + getDestination(), getOffset(), + /*requireImmediateOffset=*/true))) + return failure(); + if (getUpdatedBase() && + getUpdatedBase().getType() != getDestination().getType()) + return emitOpError("requires updated base result to match base type"); + return success(); } void SprstsOp::getEffects( @@ -4938,9 +4943,14 @@ void SprstsOp::getEffects( } LogicalResult SprstsOp::verify() { - return verifySprStoreCommon(getOperation(), "pto.sprsts", getSpr(), - getDestination(), getOffset(), - /*requireImmediateOffset=*/false); + if (failed(verifySprStoreCommon(getOperation(), "pto.sprsts", getSpr(), + getDestination(), getOffset(), + /*requireImmediateOffset=*/false))) + return failure(); + if (getUpdatedBase() && + getUpdatedBase().getType() != getDestination().getType()) + return emitOpError("requires updated base result to match base type"); + return success(); } void VldusOp::getEffects( @@ -4959,6 +4969,11 @@ LogicalResult VldusOp::verify() { return emitOpError("requires a pointer-like source"); if (classifyMemoryRole(getSource().getType()) == MemoryRole::GM) return emitOpError("requires a UB-backed source"); + if (static_cast(getIncrement()) != static_cast(getUpdatedBase())) + return emitOpError( + "requires increment and updated base result to appear together"); + if (getUpdatedBase() && getUpdatedBase().getType() != getSource().getType()) + return emitOpError("requires updated base result to match source type"); return success(); } @@ -5309,6 +5324,9 @@ LogicalResult PldsOp::verify() { return emitOpError("requires index offset"); if (!isSupportedPredicateLoadDist(getDist())) return emitOpError("requires predicate load dist to be NORM, US, or DS"); + if (getUpdatedBase() && + getUpdatedBase().getType() != getSource().getType()) + return emitOpError("requires updated base result to match base type"); return success(); } @@ -5329,6 +5347,9 @@ LogicalResult PldiOp::verify() { return emitOpError("requires offset to be a constant index immediate"); if (!isSupportedPredicateLoadDist(getDist())) return emitOpError("requires predicate load dist to be NORM, US, or DS"); + if (getUpdatedBase() && + getUpdatedBase().getType() != getSource().getType()) + return emitOpError("requires updated base result to match base type"); return success(); } @@ -6394,6 +6415,9 @@ LogicalResult Vldsx2Op::verify() { return emitOpError("requires low/high results to share one vector type"); if (!isSupportedVldx2DistToken(getDist())) return emitOpError("requires a supported x2 load distribution token"); + if (getUpdatedBase() && + getUpdatedBase().getType() != getSource().getType()) + return emitOpError("requires updated base result to match base type"); return success(); } @@ -6541,6 +6565,9 @@ LogicalResult VsldbOp::verify() { return emitOpError("requires block_stride to be i16"); if (!getRepeatStride().getType().isSignlessInteger(16)) return emitOpError("requires repeat_stride to be i16"); + if (getUpdatedBase() && + getUpdatedBase().getType() != getSource().getType()) + return emitOpError("requires updated base result to match base type"); return success(); } @@ -6569,6 +6596,9 @@ LogicalResult PstiOp::verify() { return emitOpError("requires offset to be a constant index immediate"); if (!isSupportedPredicateStoreDist(getDist())) return emitOpError("requires predicate store dist to be NORM or PK"); + if (getUpdatedBase() && + getUpdatedBase().getType() != getDestination().getType()) + return emitOpError("requires updated base result to match base type"); return success(); } @@ -6584,6 +6614,9 @@ LogicalResult PstsOp::verify() { return emitOpError("requires index offset"); if (!isSupportedPredicateStoreDist(getDist())) return emitOpError("requires predicate store dist to be NORM or PK"); + if (getUpdatedBase() && + getUpdatedBase().getType() != getDestination().getType()) + return emitOpError("requires updated base result to match base type"); return success(); } @@ -6626,6 +6659,9 @@ LogicalResult VstasOp::verify() { return emitOpError("requires a pointer-like destination"); if (classifyMemoryRole(getDestination().getType()) == MemoryRole::GM) return emitOpError("requires a UB-backed destination"); + if (getUpdatedBase() && + getUpdatedBase().getType() != getDestination().getType()) + return emitOpError("requires updated base result to match base type"); return success(); } @@ -6694,6 +6730,8 @@ LogicalResult VstusOp::verify() { return emitOpError("requires a pointer-like base"); if (classifyMemoryRole(getBase().getType()) == MemoryRole::GM) return emitOpError("requires a UB-backed base"); + if (getBaseOut() && getBaseOut().getType() != getBase().getType()) + return emitOpError("requires updated base result to match base type"); return success(); } diff --git a/lib/PTO/Transforms/VMIToVPTO.cpp b/lib/PTO/Transforms/VMIToVPTO.cpp index 79dfa5a93a..9f5e7d0c17 100644 --- a/lib/PTO/Transforms/VMIToVPTO.cpp +++ b/lib/PTO/Transforms/VMIToVPTO.cpp @@ -5840,6 +5840,7 @@ struct OneToNVMILoadOpPattern : OpConversionPattern { Value chunkOffset = createChunkOffset( op.getLoc(), *offset, group * 2 * *lanesPerPart, rewriter); auto load = rewriter.create(op.getLoc(), lowType, highType, + /*updated_base=*/Type{}, *source, chunkOffset, rewriter.getStringAttr(*dist)); lows.push_back(load.getLow()); @@ -5885,10 +5886,12 @@ struct OneToNVMILoadOpPattern : OpConversionPattern { Value secondOffset = createChunkOffset( op.getLoc(), *offset, (group * 4 + 2) * *lanesPerPart, rewriter); auto first = rewriter.create( - op.getLoc(), part0Type, part1Type, *source, firstOffset, + op.getLoc(), part0Type, part1Type, /*updated_base=*/Type{}, + *source, firstOffset, rewriter.getStringAttr(*dist)); auto second = rewriter.create( - op.getLoc(), part2Type, part3Type, *source, secondOffset, + op.getLoc(), part2Type, part3Type, /*updated_base=*/Type{}, + *source, secondOffset, rewriter.getStringAttr(*dist)); auto even = @@ -6004,8 +6007,9 @@ struct OneToNVMIDeinterleaveLoadOpPattern op.getLoc(), *offset, static_cast(index) * 2 * *lanesPerPart, rewriter); auto load = - rewriter.create(op.getLoc(), lowType, highType, *source, - chunkOffset, rewriter.getStringAttr(*dist)); + rewriter.create( + op.getLoc(), lowType, highType, /*updated_base=*/Type{}, *source, + chunkOffset, rewriter.getStringAttr(*dist)); lows.push_back(load.getLow()); highs.push_back(load.getHigh()); } @@ -6131,6 +6135,7 @@ struct OneToNVMIGroupLoadOpPattern : OpConversionPattern { Value chunkBase = makePtr(chunkOffset); results.push_back(rewriter .create(op.getLoc(), vregType, + /*updated_base=*/Type{}, chunkBase, blockStride, zeroI16, *allMask) .getResult()); @@ -6293,7 +6298,8 @@ static LogicalResult lowerGroupSlotLoadParts( createChunkOffset(op->getLoc(), offset, groupBegin, rewriter); Value slotBase = makePtr(groupOffset); results.push_back(rewriter - .create(op->getLoc(), vregType, slotBase, + .create(op->getLoc(), vregType, + /*updated_base=*/Type{}, slotBase, zeroI16, zeroI16, *slotMask) .getResult()); } @@ -6348,7 +6354,8 @@ static LogicalResult lowerGroupSlotLoadParts( } Value slotBase = makePtr(groupOffset); results.push_back(rewriter - .create(op->getLoc(), vregType, slotBase, + .create(op->getLoc(), vregType, + /*updated_base=*/Type{}, slotBase, zeroI16, zeroI16, *oneBlockMask) .getResult()); } @@ -8166,7 +8173,8 @@ struct OneToNVMIStrideLoadOpPattern .getResult(); Value loaded = rewriter - .create(op.getLoc(), resultType, base, *blockStride, + .create(op.getLoc(), resultType, + /*updated_base=*/Type{}, base, *blockStride, *repeatStride, maskParts.front()) .getResult(); replaceOpWithFlatConvertedValues(rewriter, op, SmallVector{loaded}, diff --git a/lib/PTO/Transforms/VPTOCANN900LLVMEmitter.cpp b/lib/PTO/Transforms/VPTOCANN900LLVMEmitter.cpp index 9c50f1ec7e..624100e06c 100644 --- a/lib/PTO/Transforms/VPTOCANN900LLVMEmitter.cpp +++ b/lib/PTO/Transforms/VPTOCANN900LLVMEmitter.cpp @@ -606,6 +606,8 @@ static std::string getLowPrecisionElementFragment(Type type) { } static std::string getMemoryElementTypeFragment(Type type) { + if (auto intType = dyn_cast(type)) + return "i" + std::to_string(intType.getWidth()); if (pto::isPTOHiFloat8Type(type)) return "s8"; if (std::string elem = getElementTypeFragment(type); !elem.empty()) @@ -2389,6 +2391,18 @@ static StringRef buildVstusCallee(MLIRContext *context) { return StringAttr::get(context, "llvm.hivm.vstus").getValue(); } +static FailureOr buildVstusPostCallee(MLIRContext *context, + Type valueType) { + std::string vec = + getMemoryElementTypeFragment(getElementTypeFromVectorLike(valueType)); + auto lanes = getElementCountFromVectorLike(valueType); + if (vec.empty() || !lanes) + return failure(); + return StringAttr::get(context, "llvm.hivm.vstus.post.v" + + std::to_string(*lanes) + vec) + .getValue(); +} + static StringRef buildVsturCallee(MLIRContext *context) { return StringAttr::get(context, "llvm.hivm.vstur").getValue(); } @@ -2519,25 +2533,31 @@ static StringRef buildSprclrCallee(MLIRContext *context) { return StringAttr::get(context, "llvm.hivm.sprclr").getValue(); } -static StringRef buildSprstiCallee(MLIRContext *context) { - return StringAttr::get(context, "llvm.hivm.sprsti").getValue(); +static StringRef buildSprstiCallee(MLIRContext *context, bool post) { + return StringAttr::get(context, + post ? "llvm.hivm.sprsti.post" + : "llvm.hivm.sprsti") + .getValue(); } -static StringRef buildSprstsCallee(MLIRContext *context) { - return StringAttr::get(context, "llvm.hivm.sprsts").getValue(); +static StringRef buildSprstsCallee(MLIRContext *context, bool post) { + return StringAttr::get(context, + post ? "llvm.hivm.sprsts.post" + : "llvm.hivm.sprsts") + .getValue(); } template -static StringRef buildSprStoreCallee(MLIRContext *context); +static StringRef buildSprStoreCallee(MLIRContext *context, bool post); template <> -StringRef buildSprStoreCallee(MLIRContext *context) { - return buildSprstiCallee(context); +StringRef buildSprStoreCallee(MLIRContext *context, bool post) { + return buildSprstiCallee(context, post); } template <> -StringRef buildSprStoreCallee(MLIRContext *context) { - return buildSprstsCallee(context); +StringRef buildSprStoreCallee(MLIRContext *context, bool post) { + return buildSprstsCallee(context, post); } template @@ -2568,8 +2588,11 @@ static StringRef buildVstarCallee(MLIRContext *context) { return StringAttr::get(context, "llvm.hivm.vstar").getValue(); } -static StringRef buildVstasCallee(MLIRContext *context) { - return StringAttr::get(context, "llvm.hivm.vstas").getValue(); +static StringRef buildVstasCallee(MLIRContext *context, bool post) { + return StringAttr::get(context, + post ? "llvm.hivm.vstas.post" + : "llvm.hivm.vstas") + .getValue(); } template @@ -3116,6 +3139,18 @@ static FailureOr buildVldusCallee(MLIRContext *context, .getValue(); } +static FailureOr buildVldusPostCallee(MLIRContext *context, + Type resultType) { + std::string vec = + getMemoryElementTypeFragment(getElementTypeFromVectorLike(resultType)); + auto lanes = getElementCountFromVectorLike(resultType); + if (vec.empty() || !lanes) + return failure(); + return StringAttr::get(context, "llvm.hivm.vldus.post.v" + + std::to_string(*lanes) + vec) + .getValue(); +} + static FailureOr buildVcmpCallee(MLIRContext *context, Type inputType, StringRef cmpMode, bool isScalarCompare) { @@ -3452,20 +3487,32 @@ static StringRef buildCopyCbufToFbufCallee(MLIRContext *context) { return StringAttr::get(context, "llvm.hivm.MOV.L1.TO.FB.v220").getValue(); } -static StringRef buildPstiCallee(MLIRContext *context) { - return StringAttr::get(context, "llvm.hivm.psti.b8").getValue(); +static StringRef buildPstiCallee(MLIRContext *context, bool post) { + return StringAttr::get(context, + post ? "llvm.hivm.psti.post.b8" + : "llvm.hivm.psti.b8") + .getValue(); } -static StringRef buildPstsCallee(MLIRContext *context) { - return StringAttr::get(context, "llvm.hivm.psts.b8").getValue(); +static StringRef buildPstsCallee(MLIRContext *context, bool post) { + return StringAttr::get(context, + post ? "llvm.hivm.psts.post.b8" + : "llvm.hivm.psts.b8") + .getValue(); } -static StringRef buildPldiCallee(MLIRContext *context) { - return StringAttr::get(context, "llvm.hivm.pldi.b8").getValue(); +static StringRef buildPldiCallee(MLIRContext *context, bool post) { + return StringAttr::get(context, + post ? "llvm.hivm.pldi.post.b8" + : "llvm.hivm.pldi.b8") + .getValue(); } -static StringRef buildPldsCallee(MLIRContext *context) { - return StringAttr::get(context, "llvm.hivm.plds.b8").getValue(); +static StringRef buildPldsCallee(MLIRContext *context, bool post) { + return StringAttr::get(context, + post ? "llvm.hivm.plds.post.b8" + : "llvm.hivm.plds.b8") + .getValue(); } static StringRef buildPnotCallee(MLIRContext *context) { @@ -3583,29 +3630,31 @@ static FailureOr buildVmullCallee(MLIRContext *context, } template -static StringRef getPredicateStoreCallee(MLIRContext *context); +static StringRef getPredicateStoreCallee(MLIRContext *context, bool post); template <> -StringRef getPredicateStoreCallee(MLIRContext *context) { - return buildPstiCallee(context); +StringRef getPredicateStoreCallee(MLIRContext *context, + bool post) { + return buildPstiCallee(context, post); } template <> -StringRef getPredicateStoreCallee(MLIRContext *context) { - return buildPstsCallee(context); +StringRef getPredicateStoreCallee(MLIRContext *context, + bool post) { + return buildPstsCallee(context, post); } template -static StringRef getPredicateLoadCallee(MLIRContext *context); +static StringRef getPredicateLoadCallee(MLIRContext *context, bool post); template <> -StringRef getPredicateLoadCallee(MLIRContext *context) { - return buildPldiCallee(context); +StringRef getPredicateLoadCallee(MLIRContext *context, bool post) { + return buildPldiCallee(context, post); } template <> -StringRef getPredicateLoadCallee(MLIRContext *context) { - return buildPldsCallee(context); +StringRef getPredicateLoadCallee(MLIRContext *context, bool post) { + return buildPldsCallee(context, post); } template @@ -3733,14 +3782,16 @@ static FailureOr buildVldsCallee(MLIRContext *context, Type resultTyp } static FailureOr buildVldsx2Callee(MLIRContext *context, - Type resultType) { + Type resultType, bool post) { std::string vec = getMemoryElementTypeFragment(getElementTypeFromVectorLike(resultType)); auto lanes = getElementCountFromVectorLike(resultType); if (vec.empty() || !lanes) return failure(); - return StringAttr::get(context, "llvm.hivm.vldsx2.v" + - std::to_string(*lanes) + vec) + return StringAttr::get( + context, "llvm.hivm.vldsx2" + + std::string(post ? ".post" : "") + ".v" + + std::to_string(*lanes) + vec) .getValue(); } @@ -3770,9 +3821,9 @@ buildBlockStridedMemoryCallee(MLIRContext *context, Type vectorType, } static FailureOr buildVsldbCallee(MLIRContext *context, - Type resultType) { + Type resultType, bool post) { return buildBlockStridedMemoryCallee(context, resultType, "vsldb", - /*post=*/false); + post); } static FailureOr buildVstsCallee(MLIRContext *context, Type valueType) { @@ -3787,13 +3838,17 @@ static FailureOr buildVstsCallee(MLIRContext *context, Type valueType } static FailureOr buildVstsx2Callee(MLIRContext *context, Type valueType) { - std::string vec = - getMemoryElementTypeFragment(getElementTypeFromVectorLike(valueType)); + Type elementType = getElementTypeFromVectorLike(valueType); auto lanes = getElementCountFromVectorLike(valueType); - if (vec.empty() || !lanes) + if (!elementType || !lanes) + return failure(); + + std::string element = getMemoryElementTypeFragment(elementType); + if (element.empty()) return failure(); + return StringAttr::get(context, "llvm.hivm.vstsx2.v" + - std::to_string(*lanes) + vec) + std::to_string(*lanes) + element) .getValue(); } @@ -6853,10 +6908,11 @@ class LowerVldsx2OpPattern final : public OpConversionPattern { "failed to materialize vldsx2 operands"); } + bool usePostIntrinsic = op.getUpdatedBase() != nullptr; SmallVector resultTypes; if (failed(this->getTypeConverter()->convertTypes(op->getResultTypes(), resultTypes)) || - resultTypes.size() != 2) { + resultTypes.size() != (usePostIntrinsic ? 3u : 2u)) { return rewriter.notifyMatchFailure(op, "failed to convert vldsx2 result types"); } @@ -6865,19 +6921,23 @@ class LowerVldsx2OpPattern final : public OpConversionPattern { Type highCallType = getPayloadABIType( op.getHigh().getType(), resultTypes[1], rewriter.getContext()); SmallVector callResultTypes{lowCallType, highCallType}; + if (usePostIntrinsic) + callResultTypes.push_back(resultTypes[2]); FailureOr calleeName = - buildVldsx2Callee(op.getContext(), op.getLow().getType()); + buildVldsx2Callee(op.getContext(), op.getLow().getType(), + usePostIntrinsic); if (failed(calleeName)) return rewriter.notifyMatchFailure(op, "unsupported vldsx2 signature"); Value distValue = getI32Constant(rewriter, op.getLoc(), *dist); - Value zeroValue = getI32Constant(rewriter, op.getLoc(), 0); + Value postValue = + getI32Constant(rewriter, op.getLoc(), usePostIntrinsic ? 1 : 0); SmallVector args{adaptor.getSource(), *offsetBytes, distValue, - zeroValue}; + postValue}; auto funcType = rewriter.getFunctionType( TypeRange{adaptor.getSource().getType(), (*offsetBytes).getType(), - distValue.getType(), zeroValue.getType()}, + distValue.getType(), postValue.getType()}, callResultTypes); auto call = rewriter.create(op.getLoc(), *calleeName, callResultTypes, args); @@ -6888,7 +6948,10 @@ class LowerVldsx2OpPattern final : public OpConversionPattern { Value high = castFromPayloadABI( op.getLoc(), call.getResult(1), op.getHigh().getType(), resultTypes[1], rewriter); - rewriter.replaceOp(op, ValueRange{low, high}); + if (usePostIntrinsic) + rewriter.replaceOp(op, ValueRange{low, high, call.getResult(2)}); + else + rewriter.replaceOp(op, ValueRange{low, high}); return success(); } @@ -6911,31 +6974,42 @@ class LowerVsldbOpPattern final : public OpConversionPattern { if (!basePtr || !packedStride) return rewriter.notifyMatchFailure(op, "failed to materialize vsldb operands"); - Type resultType = this->getTypeConverter()->convertType(op.getResult().getType()); - if (!resultType) + bool usePostIntrinsic = op.getUpdatedBase() != nullptr; + SmallVector resultTypes; + if (failed(this->getTypeConverter()->convertTypes(op->getResultTypes(), + resultTypes)) || + resultTypes.size() != (usePostIntrinsic ? 2u : 1u)) return rewriter.notifyMatchFailure(op, "failed to convert vsldb result type"); Type callResultType = getPayloadABIType( - op.getResult().getType(), resultType, rewriter.getContext()); + op.getResult().getType(), resultTypes[0], rewriter.getContext()); + SmallVector callResultTypes{callResultType}; + if (usePostIntrinsic) + callResultTypes.push_back(resultTypes[1]); FailureOr calleeName = - buildVsldbCallee(op.getContext(), op.getResult().getType()); + buildVsldbCallee(op.getContext(), op.getResult().getType(), + usePostIntrinsic); if (failed(calleeName)) return rewriter.notifyMatchFailure(op, "unsupported vsldb signature"); - Value zeroValue = getI32Constant(rewriter, op.getLoc(), 0); - SmallVector args{adaptor.getSource(), packedStride, zeroValue, + Value postValue = + getI32Constant(rewriter, op.getLoc(), usePostIntrinsic ? 1 : 0); + SmallVector args{adaptor.getSource(), packedStride, postValue, adaptor.getMask()}; auto funcType = rewriter.getFunctionType( TypeRange{adaptor.getSource().getType(), packedStride.getType(), - zeroValue.getType(), adaptor.getMask().getType()}, - TypeRange{callResultType}); + postValue.getType(), adaptor.getMask().getType()}, + callResultTypes); auto call = rewriter.create(op.getLoc(), *calleeName, - TypeRange{callResultType}, args); + callResultTypes, args); state.plannedDecls.push_back(PlannedDecl{calleeName->str(), funcType}); Value result = castFromPayloadABI( - op.getLoc(), call.getResult(0), op.getResult().getType(), resultType, + op.getLoc(), call.getResult(0), op.getResult().getType(), resultTypes[0], rewriter); - rewriter.replaceOp(op, ValueRange{result}); + if (usePostIntrinsic) + rewriter.replaceOp(op, ValueRange{result, call.getResult(1)}); + else + rewriter.replaceOp(op, ValueRange{result}); return success(); } @@ -7014,15 +7088,20 @@ class LowerVldusOpPattern final : public OpConversionPattern { ConversionPatternRewriter &rewriter) const override { auto sourceType = dyn_cast(adaptor.getSource().getType()); SmallVector resultTypes; + bool usePostIntrinsic = static_cast(op.getUpdatedBase()); if (!sourceType || failed(this->getTypeConverter()->convertTypes(op->getResultTypes(), resultTypes)) || - resultTypes.size() != 2 || adaptor.getAlign().getType() != resultTypes[1]) { + resultTypes.size() != (usePostIntrinsic ? 3u : 2u) || + adaptor.getAlign().getType() != resultTypes[1] || + (usePostIntrinsic && resultTypes[2] != adaptor.getSource().getType())) { return rewriter.notifyMatchFailure(op, "expected converted vldus operand/result types"); } FailureOr calleeName = - buildVldusCallee(op.getContext(), op.getResult().getType()); + usePostIntrinsic + ? buildVldusPostCallee(op.getContext(), op.getResult().getType()) + : buildVldusCallee(op.getContext(), op.getResult().getType()); if (failed(calleeName)) return rewriter.notifyMatchFailure(op, "unsupported vldus signature"); @@ -7032,17 +7111,30 @@ class LowerVldusOpPattern final : public OpConversionPattern { // The installed no-post A5 vldus intrinsic returns an extra hidden base ptr. intrinsicResultTypes.push_back(adaptor.getSource().getType()); - auto funcType = rewriter.getFunctionType( - TypeRange{adaptor.getSource().getType(), adaptor.getAlign().getType()}, - intrinsicResultTypes); + SmallVector args{adaptor.getSource(), adaptor.getAlign()}; + if (usePostIntrinsic) { + Type elementType = getElementTypeFromVectorLike(op.getResult().getType()); + auto incrementBytes = + convertElementOffsetToBytes(op, adaptor.getIncrement(), elementType); + if (failed(incrementBytes)) + return rewriter.notifyMatchFailure(op, + "failed to convert vldus increment"); + args.push_back(*incrementBytes); + } + SmallVector argTypes; + for (Value arg : args) + argTypes.push_back(arg.getType()); + auto funcType = rewriter.getFunctionType(argTypes, intrinsicResultTypes); auto call = rewriter.create( - op.getLoc(), *calleeName, intrinsicResultTypes, - ValueRange{adaptor.getSource(), adaptor.getAlign()}); + op.getLoc(), *calleeName, intrinsicResultTypes, args); state.plannedDecls.push_back(PlannedDecl{calleeName->str(), funcType}); Value loaded = castFromPayloadABI( op.getLoc(), call.getResult(0), op.getResult().getType(), resultTypes[0], rewriter); - rewriter.replaceOp(op, ValueRange{loaded, call.getResult(1)}); + SmallVector replacements{loaded, call.getResult(1)}; + if (usePostIntrinsic) + replacements.push_back(call.getResult(2)); + rewriter.replaceOp(op, replacements); return success(); } @@ -7097,20 +7189,33 @@ class LowerSprStoreOpPattern final : public OpConversionPattern { return rewriter.notifyMatchFailure(op, "expected converted spr store operands"); - StringRef calleeName = buildSprStoreCallee(op.getContext()); + bool usePostIntrinsic = op.getUpdatedBase() != nullptr; + SmallVector resultTypes; + if (failed(this->getTypeConverter()->convertTypes(op->getResultTypes(), + resultTypes)) || + resultTypes.size() != (usePostIntrinsic ? 1u : 0u)) + return rewriter.notifyMatchFailure( + op, "failed to convert spr store result types"); + + StringRef calleeName = + buildSprStoreCallee(op.getContext(), usePostIntrinsic); Value sprValue = rewriter.create( op.getLoc(), rewriter.getI16IntegerAttr(*spr)); Value postValue = rewriter.create( - op.getLoc(), rewriter.getI32IntegerAttr(0)); + op.getLoc(), rewriter.getI32IntegerAttr(usePostIntrinsic ? 1 : 0)); SmallVector args{sprValue, adaptor.getDestination(), adaptor.getOffset(), postValue}; auto funcType = rewriter.getFunctionType( TypeRange{sprValue.getType(), adaptor.getDestination().getType(), adaptor.getOffset().getType(), postValue.getType()}, - TypeRange{}); - rewriter.create(op.getLoc(), calleeName, TypeRange{}, args); + resultTypes); + auto call = rewriter.create(op.getLoc(), calleeName, + resultTypes, args); state.plannedDecls.push_back(PlannedDecl{calleeName.str(), funcType}); - rewriter.eraseOp(op); + if (usePostIntrinsic) + rewriter.replaceOp(op, call.getResults()); + else + rewriter.eraseOp(op); return success(); } @@ -7365,14 +7470,26 @@ class LowerVstusOpPattern final : public OpConversionPattern { if (failed(offsetBytes)) return rewriter.notifyMatchFailure(op, "failed to convert vstus offset"); - Type resultType = this->getTypeConverter()->convertType(op.getAlignOut().getType()); + SmallVector resultTypes; + if (failed(this->getTypeConverter()->convertTypes(op->getResultTypes(), + resultTypes))) + return rewriter.notifyMatchFailure(op, + "failed to convert vstus result types"); + bool usePostIntrinsic = static_cast(op.getBaseOut()); auto baseType = dyn_cast(adaptor.getBase().getType()); - if (!resultType || !baseType || adaptor.getAlignIn().getType() != resultType) { + if (!baseType || resultTypes.size() != (usePostIntrinsic ? 2u : 1u) || + adaptor.getAlignIn().getType() != resultTypes[0] || + (usePostIntrinsic && resultTypes[1] != adaptor.getBase().getType())) { return rewriter.notifyMatchFailure(op, "unexpected converted vstus operand/result types"); } - StringRef calleeName = buildVstusCallee(op.getContext()); + FailureOr calleeName = buildVstusCallee(op.getContext()); + if (usePostIntrinsic) + calleeName = + buildVstusPostCallee(op.getContext(), op.getValue().getType()); + if (failed(calleeName)) + return rewriter.notifyMatchFailure(op, "unsupported vstus signature"); Value value = castToPayloadABI( op.getLoc(), adaptor.getValue(), op.getValue().getType(), rewriter); SmallVector args{value, adaptor.getBase(), *offsetBytes, @@ -7380,10 +7497,10 @@ class LowerVstusOpPattern final : public OpConversionPattern { auto funcType = rewriter.getFunctionType( TypeRange{value.getType(), adaptor.getBase().getType(), (*offsetBytes).getType(), adaptor.getAlignIn().getType()}, - TypeRange{resultType}); - auto call = - rewriter.create(op.getLoc(), calleeName, TypeRange{resultType}, args); - state.plannedDecls.push_back(PlannedDecl{calleeName.str(), funcType}); + resultTypes); + auto call = rewriter.create(op.getLoc(), *calleeName, + resultTypes, args); + state.plannedDecls.push_back(PlannedDecl{calleeName->str(), funcType}); rewriter.replaceOp(op, call.getResults()); return success(); } @@ -7490,17 +7607,31 @@ class LowerVstasOpPattern final : public OpConversionPattern { if (failed(offsetBytes)) return rewriter.notifyMatchFailure(op, "failed to convert vstas offset"); - StringRef calleeName = buildVstasCallee(op.getContext()); - Value zeroValue = getI32Constant(rewriter, op.getLoc(), 0); + bool usePostIntrinsic = op.getUpdatedBase() != nullptr; + SmallVector resultTypes; + if (failed(this->getTypeConverter()->convertTypes(op->getResultTypes(), + resultTypes)) || + resultTypes.size() != (usePostIntrinsic ? 1u : 0u)) + return rewriter.notifyMatchFailure( + op, "failed to convert vstas result types"); + + StringRef calleeName = + buildVstasCallee(op.getContext(), usePostIntrinsic); + Value postValue = + getI32Constant(rewriter, op.getLoc(), usePostIntrinsic ? 1 : 0); SmallVector args{adaptor.getValue(), adaptor.getDestination(), *offsetBytes, - zeroValue}; + postValue}; auto funcType = rewriter.getFunctionType( TypeRange{adaptor.getValue().getType(), adaptor.getDestination().getType(), - (*offsetBytes).getType(), zeroValue.getType()}, - TypeRange{}); - rewriter.create(op.getLoc(), calleeName, TypeRange{}, args); + (*offsetBytes).getType(), postValue.getType()}, + resultTypes); + auto call = rewriter.create(op.getLoc(), calleeName, + resultTypes, args); state.plannedDecls.push_back(PlannedDecl{calleeName.str(), funcType}); - rewriter.eraseOp(op); + if (usePostIntrinsic) + rewriter.replaceOp(op, call.getResults()); + else + rewriter.eraseOp(op); return success(); } @@ -8153,7 +8284,16 @@ class LowerPredicateStoreOpPattern final : public OpConversionPattern { return rewriter.notifyMatchFailure( op, "failed to convert predicate-store offset to i32"); - StringRef calleeName = getPredicateStoreCallee(op.getContext()); + bool usePostIntrinsic = op.getUpdatedBase() != nullptr; + SmallVector resultTypes; + if (failed(this->getTypeConverter()->convertTypes(op->getResultTypes(), + resultTypes)) || + resultTypes.size() != (usePostIntrinsic ? 1u : 0u)) + return rewriter.notifyMatchFailure( + op, "failed to convert predicate-store result types"); + + StringRef calleeName = + getPredicateStoreCallee(op.getContext(), usePostIntrinsic); SmallVector args; args.push_back(adaptor.getValue()); args.push_back(adaptor.getDestination()); @@ -8161,14 +8301,19 @@ class LowerPredicateStoreOpPattern final : public OpConversionPattern { args.push_back(rewriter.create( op.getLoc(), rewriter.getI32IntegerAttr(*dist))); args.push_back(rewriter.create( - op.getLoc(), rewriter.getI32IntegerAttr(0))); + op.getLoc(), + rewriter.getI32IntegerAttr(usePostIntrinsic ? 1 : 0))); auto funcType = rewriter.getFunctionType( TypeRange{valueType, llvmDestType, rewriter.getI32Type(), rewriter.getI32Type(), rewriter.getI32Type()}, - TypeRange{}); - rewriter.create(op.getLoc(), calleeName, TypeRange{}, args); + resultTypes); + auto call = rewriter.create(op.getLoc(), calleeName, + resultTypes, args); state.plannedDecls.push_back(PlannedDecl{calleeName.str(), funcType}); - rewriter.eraseOp(op); + if (usePostIntrinsic) + rewriter.replaceOp(op, call.getResults()); + else + rewriter.eraseOp(op); return success(); } @@ -8189,9 +8334,14 @@ class LowerPredicateLoadOpPattern final : public OpConversionPattern { ConversionPatternRewriter &rewriter) const override { auto llvmSourceType = dyn_cast(adaptor.getSource().getType()); - Type resultType = - this->getTypeConverter()->convertType(op.getResult().getType()); - if (!llvmSourceType || !resultType) + bool usePostIntrinsic = op.getUpdatedBase() != nullptr; + SmallVector resultTypes; + if (failed(this->getTypeConverter()->convertTypes(op->getResultTypes(), + resultTypes)) || + resultTypes.size() != (usePostIntrinsic ? 2u : 1u)) + return rewriter.notifyMatchFailure( + op, "failed to convert predicate-load result types"); + if (!llvmSourceType) return rewriter.notifyMatchFailure( op, "expected converted predicate-load operand/result types"); @@ -8205,20 +8355,22 @@ class LowerPredicateLoadOpPattern final : public OpConversionPattern { return rewriter.notifyMatchFailure( op, "failed to convert predicate-load offset to i32"); - StringRef calleeName = getPredicateLoadCallee(op.getContext()); + StringRef calleeName = + getPredicateLoadCallee(op.getContext(), usePostIntrinsic); SmallVector args; args.push_back(adaptor.getSource()); args.push_back(offset); args.push_back(rewriter.create( op.getLoc(), rewriter.getI32IntegerAttr(*dist))); args.push_back(rewriter.create( - op.getLoc(), rewriter.getI32IntegerAttr(0))); + op.getLoc(), + rewriter.getI32IntegerAttr(usePostIntrinsic ? 1 : 0))); auto funcType = rewriter.getFunctionType( TypeRange{llvmSourceType, rewriter.getI32Type(), rewriter.getI32Type(), rewriter.getI32Type()}, - TypeRange{resultType}); - auto call = - rewriter.create(op.getLoc(), calleeName, resultType, args); + resultTypes); + auto call = rewriter.create(op.getLoc(), calleeName, + resultTypes, args); state.plannedDecls.push_back(PlannedDecl{calleeName.str(), funcType}); rewriter.replaceOp(op, call.getResults()); return success(); diff --git a/lib/PTO/Transforms/VPTOLLVMEmitter.cpp b/lib/PTO/Transforms/VPTOLLVMEmitter.cpp index e7d5a79906..72fc8e6e59 100644 --- a/lib/PTO/Transforms/VPTOLLVMEmitter.cpp +++ b/lib/PTO/Transforms/VPTOLLVMEmitter.cpp @@ -519,22 +519,6 @@ static FailureOr buildLaneTypedCalleeFromInput(MLIRContext *context, .getValue(); } -static FailureOr buildMemoryLaneTypedCallee(MLIRContext *context, - Type valueType, - StringRef stem, - StringRef suffix) { - std::string vec = - getMemoryElementTypeFragment(getElementTypeFromVectorLike(valueType)); - auto lanes = getElementCountFromVectorLike(valueType); - if (vec.empty() || !lanes) - return failure(); - - return StringAttr::get(context, "llvm.hivm." + stem.str() + ".v" + - std::to_string(*lanes) + vec + - suffix.str()) - .getValue(); -} - static std::string getElementTypeFragment(Type type) { if (type.isF16()) return "f16"; @@ -564,6 +548,8 @@ static std::string getLowPrecisionElementFragment(Type type) { } static std::string getMemoryElementTypeFragment(Type type) { + if (auto intType = dyn_cast(type)) + return "i" + std::to_string(intType.getWidth()); if (std::string elem = getElementTypeFragment(type); !elem.empty()) return elem; return getLowPrecisionElementFragment(type); @@ -2433,6 +2419,18 @@ static StringRef buildVstusCallee(MLIRContext *context) { return StringAttr::get(context, "llvm.hivm.vstus").getValue(); } +static FailureOr buildVstusPostCallee(MLIRContext *context, + Type valueType) { + std::string vec = + getMemoryElementTypeFragment(getElementTypeFromVectorLike(valueType)); + auto lanes = getElementCountFromVectorLike(valueType); + if (vec.empty() || !lanes) + return failure(); + return StringAttr::get(context, "llvm.hivm.vstus.post.v" + + std::to_string(*lanes) + vec) + .getValue(); +} + static StringRef buildVsturCallee(MLIRContext *context) { return StringAttr::get(context, "llvm.hivm.vstur").getValue(); } @@ -2563,25 +2561,31 @@ static StringRef buildSprclrCallee(MLIRContext *context) { return StringAttr::get(context, "llvm.hivm.sprclr").getValue(); } -static StringRef buildSprstiCallee(MLIRContext *context) { - return StringAttr::get(context, "llvm.hivm.sprsti").getValue(); +static StringRef buildSprstiCallee(MLIRContext *context, bool post) { + return StringAttr::get(context, + post ? "llvm.hivm.sprsti.post" + : "llvm.hivm.sprsti") + .getValue(); } -static StringRef buildSprstsCallee(MLIRContext *context) { - return StringAttr::get(context, "llvm.hivm.sprsts").getValue(); +static StringRef buildSprstsCallee(MLIRContext *context, bool post) { + return StringAttr::get(context, + post ? "llvm.hivm.sprsts.post" + : "llvm.hivm.sprsts") + .getValue(); } template -static StringRef buildSprStoreCallee(MLIRContext *context); +static StringRef buildSprStoreCallee(MLIRContext *context, bool post); template <> -StringRef buildSprStoreCallee(MLIRContext *context) { - return buildSprstiCallee(context); +StringRef buildSprStoreCallee(MLIRContext *context, bool post) { + return buildSprstiCallee(context, post); } template <> -StringRef buildSprStoreCallee(MLIRContext *context) { - return buildSprstsCallee(context); +StringRef buildSprStoreCallee(MLIRContext *context, bool post) { + return buildSprstsCallee(context, post); } template @@ -2612,8 +2616,11 @@ static StringRef buildVstarCallee(MLIRContext *context) { return StringAttr::get(context, "llvm.hivm.vstar").getValue(); } -static StringRef buildVstasCallee(MLIRContext *context) { - return StringAttr::get(context, "llvm.hivm.vstas").getValue(); +static StringRef buildVstasCallee(MLIRContext *context, bool post) { + return StringAttr::get(context, + post ? "llvm.hivm.vstas.post" + : "llvm.hivm.vstas") + .getValue(); } template @@ -3160,6 +3167,18 @@ static FailureOr buildVldusCallee(MLIRContext *context, .getValue(); } +static FailureOr buildVldusPostCallee(MLIRContext *context, + Type resultType) { + std::string vec = + getMemoryElementTypeFragment(getElementTypeFromVectorLike(resultType)); + auto lanes = getElementCountFromVectorLike(resultType); + if (vec.empty() || !lanes) + return failure(); + return StringAttr::get(context, "llvm.hivm.vldus.post.v" + + std::to_string(*lanes) + vec) + .getValue(); +} + static FailureOr buildVcmpCallee(MLIRContext *context, Type inputType, StringRef cmpMode, bool isScalarCompare) { @@ -3504,20 +3523,32 @@ static StringRef buildCopyCbufToFbufCallee(MLIRContext *context) { return StringAttr::get(context, "llvm.hivm.MOV.L1.TO.FB.v220").getValue(); } -static StringRef buildPstiCallee(MLIRContext *context) { - return StringAttr::get(context, "llvm.hivm.psti.b8").getValue(); +static StringRef buildPstiCallee(MLIRContext *context, bool post) { + return StringAttr::get(context, + post ? "llvm.hivm.psti.post.b8" + : "llvm.hivm.psti.b8") + .getValue(); } -static StringRef buildPstsCallee(MLIRContext *context) { - return StringAttr::get(context, "llvm.hivm.psts.b8").getValue(); +static StringRef buildPstsCallee(MLIRContext *context, bool post) { + return StringAttr::get(context, + post ? "llvm.hivm.psts.post.b8" + : "llvm.hivm.psts.b8") + .getValue(); } -static StringRef buildPldiCallee(MLIRContext *context) { - return StringAttr::get(context, "llvm.hivm.pldi.b8").getValue(); +static StringRef buildPldiCallee(MLIRContext *context, bool post) { + return StringAttr::get(context, + post ? "llvm.hivm.pldi.post.b8" + : "llvm.hivm.pldi.b8") + .getValue(); } -static StringRef buildPldsCallee(MLIRContext *context) { - return StringAttr::get(context, "llvm.hivm.plds.b8").getValue(); +static StringRef buildPldsCallee(MLIRContext *context, bool post) { + return StringAttr::get(context, + post ? "llvm.hivm.plds.post.b8" + : "llvm.hivm.plds.b8") + .getValue(); } static StringRef buildPnotCallee(MLIRContext *context) { @@ -3636,29 +3667,31 @@ static FailureOr buildVmullCallee(MLIRContext *context, } template -static StringRef getPredicateStoreCallee(MLIRContext *context); +static StringRef getPredicateStoreCallee(MLIRContext *context, bool post); template <> -StringRef getPredicateStoreCallee(MLIRContext *context) { - return buildPstiCallee(context); +StringRef getPredicateStoreCallee(MLIRContext *context, + bool post) { + return buildPstiCallee(context, post); } template <> -StringRef getPredicateStoreCallee(MLIRContext *context) { - return buildPstsCallee(context); +StringRef getPredicateStoreCallee(MLIRContext *context, + bool post) { + return buildPstsCallee(context, post); } template -static StringRef getPredicateLoadCallee(MLIRContext *context); +static StringRef getPredicateLoadCallee(MLIRContext *context, bool post); template <> -StringRef getPredicateLoadCallee(MLIRContext *context) { - return buildPldiCallee(context); +StringRef getPredicateLoadCallee(MLIRContext *context, bool post) { + return buildPldiCallee(context, post); } template <> -StringRef getPredicateLoadCallee(MLIRContext *context) { - return buildPldsCallee(context); +StringRef getPredicateLoadCallee(MLIRContext *context, bool post) { + return buildPldsCallee(context, post); } template @@ -3786,8 +3819,19 @@ static FailureOr buildVldsCallee(MLIRContext *context, Type resultTyp } static FailureOr buildVldsx2Callee(MLIRContext *context, - Type resultType) { - return buildMemoryLaneTypedCallee(context, resultType, "vldsx2", ""); + Type resultType, bool post) { + Type elementType = getElementTypeFromVectorLike(resultType); + auto lanes = getElementCountFromVectorLike(resultType); + if (!elementType || !lanes) + return failure(); + std::string element = getMemoryElementTypeFragment(elementType); + if (element.empty()) + return failure(); + return StringAttr::get( + context, "llvm.hivm.vldsx2" + + std::string(post ? ".post" : "") + ".v" + + std::to_string(*lanes) + element) + .getValue(); } static FailureOr @@ -3816,9 +3860,9 @@ buildBlockStridedMemoryCallee(MLIRContext *context, Type vectorType, } static FailureOr buildVsldbCallee(MLIRContext *context, - Type resultType) { + Type resultType, bool post) { return buildBlockStridedMemoryCallee(context, resultType, "vsldb", - /*post=*/false); + post); } static FailureOr buildVstsCallee(MLIRContext *context, Type valueType) { @@ -3833,7 +3877,18 @@ static FailureOr buildVstsCallee(MLIRContext *context, Type valueType } static FailureOr buildVstsx2Callee(MLIRContext *context, Type valueType) { - return buildMemoryLaneTypedCallee(context, valueType, "vstsx2", ""); + Type elementType = getElementTypeFromVectorLike(valueType); + auto lanes = getElementCountFromVectorLike(valueType); + if (!elementType || !lanes) + return failure(); + + std::string element = getMemoryElementTypeFragment(elementType); + if (element.empty()) + return failure(); + + return StringAttr::get(context, "llvm.hivm.vstsx2.v" + + std::to_string(*lanes) + element) + .getValue(); } static FailureOr buildVsstbCallee(MLIRContext *context, @@ -7451,16 +7506,18 @@ class LowerVldsx2OpPattern final : public OpConversionPattern { "failed to materialize vldsx2 operands"); } + bool usePostIntrinsic = op.getUpdatedBase() != nullptr; SmallVector resultTypes; if (failed(this->getTypeConverter()->convertTypes(op->getResultTypes(), resultTypes)) || - resultTypes.size() != 2) { + resultTypes.size() != (usePostIntrinsic ? 3u : 2u)) { return rewriter.notifyMatchFailure(op, "failed to convert vldsx2 result types"); } FailureOr calleeName = - buildVldsx2Callee(op.getContext(), op.getLow().getType()); + buildVldsx2Callee(op.getContext(), op.getLow().getType(), + usePostIntrinsic); if (failed(calleeName)) return rewriter.notifyMatchFailure(op, "unsupported vldsx2 signature"); @@ -7469,14 +7526,17 @@ class LowerVldsx2OpPattern final : public OpConversionPattern { Type highCallType = getPayloadABIType( op.getHigh().getType(), resultTypes[1], rewriter.getContext()); SmallVector callResultTypes{lowCallType, highCallType}; + if (usePostIntrinsic) + callResultTypes.push_back(resultTypes[2]); Value distValue = getI32Constant(rewriter, op.getLoc(), *dist); - Value zeroValue = getI32Constant(rewriter, op.getLoc(), 0); + Value postValue = + getI32Constant(rewriter, op.getLoc(), usePostIntrinsic ? 1 : 0); SmallVector args{adaptor.getSource(), *offsetBytes, distValue, - zeroValue}; + postValue}; auto funcType = rewriter.getFunctionType( TypeRange{adaptor.getSource().getType(), (*offsetBytes).getType(), - distValue.getType(), zeroValue.getType()}, + distValue.getType(), postValue.getType()}, callResultTypes); auto call = rewriter.create(op.getLoc(), *calleeName, callResultTypes, args); @@ -7487,7 +7547,10 @@ class LowerVldsx2OpPattern final : public OpConversionPattern { Value high = castFromPayloadABI( op.getLoc(), call.getResult(1), op.getHigh().getType(), resultTypes[1], rewriter); - rewriter.replaceOp(op, ValueRange{low, high}); + if (usePostIntrinsic) + rewriter.replaceOp(op, ValueRange{low, high, call.getResult(2)}); + else + rewriter.replaceOp(op, ValueRange{low, high}); return success(); } @@ -7510,31 +7573,42 @@ class LowerVsldbOpPattern final : public OpConversionPattern { if (!basePtr || !packedStride) return rewriter.notifyMatchFailure(op, "failed to materialize vsldb operands"); - Type resultType = this->getTypeConverter()->convertType(op.getResult().getType()); - if (!resultType) + bool usePostIntrinsic = op.getUpdatedBase() != nullptr; + SmallVector resultTypes; + if (failed(this->getTypeConverter()->convertTypes(op->getResultTypes(), + resultTypes)) || + resultTypes.size() != (usePostIntrinsic ? 2u : 1u)) return rewriter.notifyMatchFailure(op, "failed to convert vsldb result type"); Type callResultType = getPayloadABIType( - op.getResult().getType(), resultType, rewriter.getContext()); + op.getResult().getType(), resultTypes[0], rewriter.getContext()); + SmallVector callResultTypes{callResultType}; + if (usePostIntrinsic) + callResultTypes.push_back(resultTypes[1]); FailureOr calleeName = - buildVsldbCallee(op.getContext(), op.getResult().getType()); + buildVsldbCallee(op.getContext(), op.getResult().getType(), + usePostIntrinsic); if (failed(calleeName)) return rewriter.notifyMatchFailure(op, "unsupported vsldb signature"); - Value zeroValue = getI32Constant(rewriter, op.getLoc(), 0); - SmallVector args{adaptor.getSource(), packedStride, zeroValue, + Value postValue = + getI32Constant(rewriter, op.getLoc(), usePostIntrinsic ? 1 : 0); + SmallVector args{adaptor.getSource(), packedStride, postValue, adaptor.getMask()}; auto funcType = rewriter.getFunctionType( TypeRange{adaptor.getSource().getType(), packedStride.getType(), - zeroValue.getType(), adaptor.getMask().getType()}, - TypeRange{callResultType}); + postValue.getType(), adaptor.getMask().getType()}, + callResultTypes); auto call = rewriter.create(op.getLoc(), *calleeName, - TypeRange{callResultType}, args); + callResultTypes, args); state.plannedDecls.push_back(PlannedDecl{calleeName->str(), funcType}); Value result = castFromPayloadABI( - op.getLoc(), call.getResult(0), op.getResult().getType(), resultType, + op.getLoc(), call.getResult(0), op.getResult().getType(), resultTypes[0], rewriter); - rewriter.replaceOp(op, ValueRange{result}); + if (usePostIntrinsic) + rewriter.replaceOp(op, ValueRange{result, call.getResult(1)}); + else + rewriter.replaceOp(op, ValueRange{result}); return success(); } @@ -7613,15 +7687,20 @@ class LowerVldusOpPattern final : public OpConversionPattern { ConversionPatternRewriter &rewriter) const override { auto sourceType = dyn_cast(adaptor.getSource().getType()); SmallVector resultTypes; + bool usePostIntrinsic = static_cast(op.getUpdatedBase()); if (!sourceType || failed(this->getTypeConverter()->convertTypes(op->getResultTypes(), resultTypes)) || - resultTypes.size() != 2 || adaptor.getAlign().getType() != resultTypes[1]) { + resultTypes.size() != (usePostIntrinsic ? 3u : 2u) || + adaptor.getAlign().getType() != resultTypes[1] || + (usePostIntrinsic && resultTypes[2] != adaptor.getSource().getType())) { return rewriter.notifyMatchFailure(op, "expected converted vldus operand/result types"); } FailureOr calleeName = - buildVldusCallee(op.getContext(), op.getResult().getType()); + usePostIntrinsic + ? buildVldusPostCallee(op.getContext(), op.getResult().getType()) + : buildVldusCallee(op.getContext(), op.getResult().getType()); if (failed(calleeName)) return rewriter.notifyMatchFailure(op, "unsupported vldus signature"); @@ -7631,17 +7710,30 @@ class LowerVldusOpPattern final : public OpConversionPattern { // The installed no-post A5 vldus intrinsic returns an extra hidden base ptr. intrinsicResultTypes.push_back(adaptor.getSource().getType()); - auto funcType = rewriter.getFunctionType( - TypeRange{adaptor.getSource().getType(), adaptor.getAlign().getType()}, - intrinsicResultTypes); + SmallVector args{adaptor.getSource(), adaptor.getAlign()}; + if (usePostIntrinsic) { + Type elementType = getElementTypeFromVectorLike(op.getResult().getType()); + auto incrementBytes = + convertElementOffsetToBytes(op, adaptor.getIncrement(), elementType); + if (failed(incrementBytes)) + return rewriter.notifyMatchFailure(op, + "failed to convert vldus increment"); + args.push_back(*incrementBytes); + } + SmallVector argTypes; + for (Value arg : args) + argTypes.push_back(arg.getType()); + auto funcType = rewriter.getFunctionType(argTypes, intrinsicResultTypes); auto call = rewriter.create( - op.getLoc(), *calleeName, intrinsicResultTypes, - ValueRange{adaptor.getSource(), adaptor.getAlign()}); + op.getLoc(), *calleeName, intrinsicResultTypes, args); state.plannedDecls.push_back(PlannedDecl{calleeName->str(), funcType}); Value loaded = castFromPayloadABI( op.getLoc(), call.getResult(0), op.getResult().getType(), resultTypes[0], rewriter); - rewriter.replaceOp(op, ValueRange{loaded, call.getResult(1)}); + SmallVector replacements{loaded, call.getResult(1)}; + if (usePostIntrinsic) + replacements.push_back(call.getResult(2)); + rewriter.replaceOp(op, replacements); return success(); } @@ -7696,20 +7788,33 @@ class LowerSprStoreOpPattern final : public OpConversionPattern { return rewriter.notifyMatchFailure(op, "expected converted spr store operands"); - StringRef calleeName = buildSprStoreCallee(op.getContext()); + bool usePostIntrinsic = op.getUpdatedBase() != nullptr; + SmallVector resultTypes; + if (failed(this->getTypeConverter()->convertTypes(op->getResultTypes(), + resultTypes)) || + resultTypes.size() != (usePostIntrinsic ? 1u : 0u)) + return rewriter.notifyMatchFailure( + op, "failed to convert spr store result types"); + + StringRef calleeName = + buildSprStoreCallee(op.getContext(), usePostIntrinsic); Value sprValue = rewriter.create( op.getLoc(), rewriter.getI16IntegerAttr(*spr)); Value postValue = rewriter.create( - op.getLoc(), rewriter.getI32IntegerAttr(0)); + op.getLoc(), rewriter.getI32IntegerAttr(usePostIntrinsic ? 1 : 0)); SmallVector args{sprValue, adaptor.getDestination(), adaptor.getOffset(), postValue}; auto funcType = rewriter.getFunctionType( TypeRange{sprValue.getType(), adaptor.getDestination().getType(), adaptor.getOffset().getType(), postValue.getType()}, - TypeRange{}); - rewriter.create(op.getLoc(), calleeName, TypeRange{}, args); + resultTypes); + auto call = rewriter.create(op.getLoc(), calleeName, + resultTypes, args); state.plannedDecls.push_back(PlannedDecl{calleeName.str(), funcType}); - rewriter.eraseOp(op); + if (usePostIntrinsic) + rewriter.replaceOp(op, call.getResults()); + else + rewriter.eraseOp(op); return success(); } @@ -7964,14 +8069,26 @@ class LowerVstusOpPattern final : public OpConversionPattern { if (failed(offsetBytes)) return rewriter.notifyMatchFailure(op, "failed to convert vstus offset"); - Type resultType = this->getTypeConverter()->convertType(op.getAlignOut().getType()); + SmallVector resultTypes; + if (failed(this->getTypeConverter()->convertTypes(op->getResultTypes(), + resultTypes))) + return rewriter.notifyMatchFailure(op, + "failed to convert vstus result types"); + bool usePostIntrinsic = static_cast(op.getBaseOut()); auto baseType = dyn_cast(adaptor.getBase().getType()); - if (!resultType || !baseType || adaptor.getAlignIn().getType() != resultType) { + if (!baseType || resultTypes.size() != (usePostIntrinsic ? 2u : 1u) || + adaptor.getAlignIn().getType() != resultTypes[0] || + (usePostIntrinsic && resultTypes[1] != adaptor.getBase().getType())) { return rewriter.notifyMatchFailure(op, "unexpected converted vstus operand/result types"); } - StringRef calleeName = buildVstusCallee(op.getContext()); + FailureOr calleeName = buildVstusCallee(op.getContext()); + if (usePostIntrinsic) + calleeName = + buildVstusPostCallee(op.getContext(), op.getValue().getType()); + if (failed(calleeName)) + return rewriter.notifyMatchFailure(op, "unsupported vstus signature"); Value value = castToPayloadABI( op.getLoc(), adaptor.getValue(), op.getValue().getType(), rewriter); SmallVector args{value, adaptor.getBase(), *offsetBytes, @@ -7979,10 +8096,10 @@ class LowerVstusOpPattern final : public OpConversionPattern { auto funcType = rewriter.getFunctionType( TypeRange{value.getType(), adaptor.getBase().getType(), (*offsetBytes).getType(), adaptor.getAlignIn().getType()}, - TypeRange{resultType}); - auto call = - rewriter.create(op.getLoc(), calleeName, TypeRange{resultType}, args); - state.plannedDecls.push_back(PlannedDecl{calleeName.str(), funcType}); + resultTypes); + auto call = rewriter.create(op.getLoc(), *calleeName, + resultTypes, args); + state.plannedDecls.push_back(PlannedDecl{calleeName->str(), funcType}); rewriter.replaceOp(op, call.getResults()); return success(); } @@ -8089,17 +8206,31 @@ class LowerVstasOpPattern final : public OpConversionPattern { if (failed(offsetBytes)) return rewriter.notifyMatchFailure(op, "failed to convert vstas offset"); - StringRef calleeName = buildVstasCallee(op.getContext()); - Value zeroValue = getI32Constant(rewriter, op.getLoc(), 0); + bool usePostIntrinsic = op.getUpdatedBase() != nullptr; + SmallVector resultTypes; + if (failed(this->getTypeConverter()->convertTypes(op->getResultTypes(), + resultTypes)) || + resultTypes.size() != (usePostIntrinsic ? 1u : 0u)) + return rewriter.notifyMatchFailure( + op, "failed to convert vstas result types"); + + StringRef calleeName = + buildVstasCallee(op.getContext(), usePostIntrinsic); + Value postValue = + getI32Constant(rewriter, op.getLoc(), usePostIntrinsic ? 1 : 0); SmallVector args{adaptor.getValue(), adaptor.getDestination(), *offsetBytes, - zeroValue}; + postValue}; auto funcType = rewriter.getFunctionType( TypeRange{adaptor.getValue().getType(), adaptor.getDestination().getType(), - (*offsetBytes).getType(), zeroValue.getType()}, - TypeRange{}); - rewriter.create(op.getLoc(), calleeName, TypeRange{}, args); + (*offsetBytes).getType(), postValue.getType()}, + resultTypes); + auto call = rewriter.create(op.getLoc(), calleeName, + resultTypes, args); state.plannedDecls.push_back(PlannedDecl{calleeName.str(), funcType}); - rewriter.eraseOp(op); + if (usePostIntrinsic) + rewriter.replaceOp(op, call.getResults()); + else + rewriter.eraseOp(op); return success(); } @@ -8767,7 +8898,16 @@ class LowerPredicateStoreOpPattern final : public OpConversionPattern { return rewriter.notifyMatchFailure( op, "failed to convert predicate-store offset to i32"); - StringRef calleeName = getPredicateStoreCallee(op.getContext()); + bool usePostIntrinsic = op.getUpdatedBase() != nullptr; + SmallVector resultTypes; + if (failed(this->getTypeConverter()->convertTypes(op->getResultTypes(), + resultTypes)) || + resultTypes.size() != (usePostIntrinsic ? 1u : 0u)) + return rewriter.notifyMatchFailure( + op, "failed to convert predicate-store result types"); + + StringRef calleeName = + getPredicateStoreCallee(op.getContext(), usePostIntrinsic); SmallVector args; args.push_back(adaptor.getValue()); args.push_back(adaptor.getDestination()); @@ -8775,14 +8915,19 @@ class LowerPredicateStoreOpPattern final : public OpConversionPattern { args.push_back(rewriter.create( op.getLoc(), rewriter.getI32IntegerAttr(*dist))); args.push_back(rewriter.create( - op.getLoc(), rewriter.getI32IntegerAttr(0))); + op.getLoc(), + rewriter.getI32IntegerAttr(usePostIntrinsic ? 1 : 0))); auto funcType = rewriter.getFunctionType( TypeRange{valueType, llvmDestType, rewriter.getI32Type(), rewriter.getI32Type(), rewriter.getI32Type()}, - TypeRange{}); - rewriter.create(op.getLoc(), calleeName, TypeRange{}, args); + resultTypes); + auto call = rewriter.create(op.getLoc(), calleeName, + resultTypes, args); state.plannedDecls.push_back(PlannedDecl{calleeName.str(), funcType}); - rewriter.eraseOp(op); + if (usePostIntrinsic) + rewriter.replaceOp(op, call.getResults()); + else + rewriter.eraseOp(op); return success(); } @@ -8803,9 +8948,14 @@ class LowerPredicateLoadOpPattern final : public OpConversionPattern { ConversionPatternRewriter &rewriter) const override { auto llvmSourceType = dyn_cast(adaptor.getSource().getType()); - Type resultType = - this->getTypeConverter()->convertType(op.getResult().getType()); - if (!llvmSourceType || !resultType) + bool usePostIntrinsic = op.getUpdatedBase() != nullptr; + SmallVector resultTypes; + if (failed(this->getTypeConverter()->convertTypes(op->getResultTypes(), + resultTypes)) || + resultTypes.size() != (usePostIntrinsic ? 2u : 1u)) + return rewriter.notifyMatchFailure( + op, "failed to convert predicate-load result types"); + if (!llvmSourceType) return rewriter.notifyMatchFailure( op, "expected converted predicate-load operand/result types"); @@ -8819,20 +8969,22 @@ class LowerPredicateLoadOpPattern final : public OpConversionPattern { return rewriter.notifyMatchFailure( op, "failed to convert predicate-load offset to i32"); - StringRef calleeName = getPredicateLoadCallee(op.getContext()); + StringRef calleeName = + getPredicateLoadCallee(op.getContext(), usePostIntrinsic); SmallVector args; args.push_back(adaptor.getSource()); args.push_back(offset); args.push_back(rewriter.create( op.getLoc(), rewriter.getI32IntegerAttr(*dist))); args.push_back(rewriter.create( - op.getLoc(), rewriter.getI32IntegerAttr(0))); + op.getLoc(), + rewriter.getI32IntegerAttr(usePostIntrinsic ? 1 : 0))); auto funcType = rewriter.getFunctionType( TypeRange{llvmSourceType, rewriter.getI32Type(), rewriter.getI32Type(), rewriter.getI32Type()}, - TypeRange{resultType}); - auto call = - rewriter.create(op.getLoc(), calleeName, resultType, args); + resultTypes); + auto call = rewriter.create(op.getLoc(), calleeName, + resultTypes, args); state.plannedDecls.push_back(PlannedDecl{calleeName.str(), funcType}); rewriter.replaceOp(op, call.getResults()); return success(); diff --git a/lib/PTO/Transforms/VPTOSoftPostUpdate.cpp b/lib/PTO/Transforms/VPTOSoftPostUpdate.cpp index 72d57e10fa..217ca28d20 100644 --- a/lib/PTO/Transforms/VPTOSoftPostUpdate.cpp +++ b/lib/PTO/Transforms/VPTOSoftPostUpdate.cpp @@ -24,6 +24,7 @@ #include #include #include +#include namespace mlir { namespace pto { @@ -47,21 +48,35 @@ static constexpr unsigned kCanonicalAddressWidth = 16; // What one unit of an op's strideOperand means, in address terms. This is a // property of the op's lowering, not of the pass: `Element` ops run their // offset through convertElementOffsetToBytes, `Block` ops pass a packed -// control word straight to the intrinsic, and `Byte` ops pass a raw byte -// offset. See `strideUnitBytes` for the conversion. +// control word straight to the intrinsic, `Alignment` ops use an op-specific +// hardware alignment table, and `Byte` ops pass a raw byte offset. See +// `strideUnitBytes` for the conversion. enum class StrideUnit { - Element, // vlds/vsts/vldsx2/vstas: offset in pointer elements - Block, // vsstb/vsldb: repeat_stride in 32-byte blocks - Byte, // sprsts/sprsti: raw byte offset + Element, // vector load/store offsets and increments in pointer elements + Block, // vsstb/vsldb: offset in 32-byte blocks + Alignment, // pldi/psti/sprsti: op-specific hardware alignment units + Byte, // plds/psts/sprsts: scalar offset in bytes +}; + +enum class StrideConstraint { + Dynamic, + Constant, + SignedI8, }; // Per-op-type descriptor: how to extract address operands and check // post-update. base/strideOperand indices are operand positions. struct PostUpdateOpInfo { int baseOperandIdx; - int strideOperandIdx; + // A missing stride operand models an op whose explicit address offset is + // zero. Its post-update increment then comes entirely from base evolution. + std::optional strideOperandIdx; StrideUnit strideUnit; unsigned minResultsForPost; // numResults > this means already post-update + StrideConstraint strideConstraint = StrideConstraint::Dynamic; + // Whether strideOperand contributes to the current access address. Stateful + // stream ops may use it only as the distance advanced after the access. + bool strideIsInitialOffset = true; }; using PostUpdateTable = llvm::StringMap; @@ -71,7 +86,22 @@ static const PostUpdateTable &getPostUpdateTable() { PostUpdateTable t; // base strideOp strideUnit minResults t["pto.vlds"] = {0, 1, StrideUnit::Element, 1}; + t["pto.vldsx2"] = {0, 1, StrideUnit::Element, 2}; + t["pto.vldus"] = {0, std::nullopt, StrideUnit::Element, 2}; + t["pto.plds"] = {0, 1, StrideUnit::Byte, 1}; + t["pto.pldi"] = {0, 1, StrideUnit::Alignment, 1, + StrideConstraint::Constant}; t["pto.vsts"] = {1, 2, StrideUnit::Element, 0}; + t["pto.vstus"] = {3, 1, StrideUnit::Element, 1, + StrideConstraint::Dynamic, false}; + t["pto.psts"] = {1, 2, StrideUnit::Byte, 0}; + t["pto.psti"] = {1, 2, StrideUnit::Alignment, 0, + StrideConstraint::Constant}; + t["pto.sprsts"] = {0, 1, StrideUnit::Byte, 0}; + t["pto.sprsti"] = {0, 1, StrideUnit::Alignment, 0, + StrideConstraint::SignedI8}; + t["pto.vstas"] = {1, 2, StrideUnit::Element, 0}; + t["pto.vsldb"] = {0, 2, StrideUnit::Block, 1}; t["pto.vsstb"] = {1, 3, StrideUnit::Block, 0}; return t; }(); @@ -101,13 +131,17 @@ static std::optional addPtrUnitBytes(Value base) { return static_cast(bits / 8); } -// Bytes covered by one unit of the op's strideOperand. -static int64_t strideUnitBytes(StrideUnit unit, int64_t elemBytes) { +// Bytes covered by one unit of the op's strideOperand. Some units depend on +// op attributes, so an unknown table entry conservatively rejects the op. +static std::optional strideUnitBytes(Operation *op, StrideUnit unit, + int64_t elemBytes) { switch (unit) { case StrideUnit::Element: return elemBytes; case StrideUnit::Block: return kBlockSizeBytes; + case StrideUnit::Alignment: + return pto::getLoadStoreVecAlignmentSize(op); case StrideUnit::Byte: return 1; } @@ -126,7 +160,9 @@ static void extractBaseAndStrideOperand(Operation *op, const PostUpdateOpInfo &info, Value &base, Value &strideOperand) { base = op->getOperand(info.baseOperandIdx); - strideOperand = op->getOperand(info.strideOperandIdx); + strideOperand = info.strideOperandIdx + ? op->getOperand(*info.strideOperandIdx) + : Value(); } // Check if op already has an updated_base result. @@ -996,6 +1032,20 @@ static Value truncateElementOffsetToI32(Value offset, Location loc, offsetI32); } +// pto.addptr always consumes an index offset. Block offsets retain their +// existing unsigned interpretation; every other supported address unit is +// signed, including sprsti's signed 8-bit word offset. +static Value normalizeAddPtrOffsetToIndex(Value offset, StrideUnit strideUnit, + Location loc, OpBuilder &builder) { + if (offset.getType().isIndex()) + return offset; + if (strideUnit == StrideUnit::Block) + return builder.create(loc, builder.getIndexType(), + offset); + return builder.create(loc, builder.getIndexType(), + offset); +} + // Create the address reached by one memory op before post-update rewriting. // The builder must already point at the desired insertion location. static Value createInitialPtr(Value base, Value strideOperand, @@ -1029,6 +1079,8 @@ static Value createInitialPtr(Value base, Value strideOperand, builder.create(loc, *constSo / divisor); } } + scaledOffset = + normalizeAddPtrOffsetToIndex(scaledOffset, strideUnit, loc, builder); return builder.create(loc, base, scaledOffset); } @@ -1202,9 +1254,9 @@ static StrideExprRef makeAvailableAt(const StrideExprRef &e, // Constants are loop-invariant, so they are always emitted before the loop and // shared across every candidate in it. Sharing matters beyond tidiness: the -// rewrite groups ops by (base, stride) Value identity, so two candidates with -// the same numeric stride must end up with the *same* Value to share an -// iter_arg. +// rewrite groups ops by base/offset/stride Value identity and effective byte +// unit, so two compatible candidates with the same numeric stride must end up +// with the *same* Value to share an iter_arg. using ConstCache = DenseMap, Value>; static Value materializeConst(int64_t c, Type ty, Location loc, @@ -1254,6 +1306,17 @@ static bool constantsFitType(const StrideExprRef &e, Type wantType) { return false; } +static bool satisfiesStrideConstraint(const StrideExprRef &stride, + StrideConstraint constraint) { + if (constraint == StrideConstraint::Dynamic) + return true; + std::optional constant = foldConst(stride); + if (!constant) + return false; + return constraint == StrideConstraint::Constant || + (*constant >= -128 && *constant <= 127); +} + // Emit `e` at the builder's current insertion point. Sub-expressions are // emitted bottom-up, so every operand is created before its user and the // result dominates the insertion point by construction. @@ -1425,6 +1488,7 @@ struct PostUpdateRewrite { Value strideOperand; // original offset / repeat_stride operand Value stride; // stride value (stride_new for block-stride ops) Value initPtr; // base + strideOperand_at_iter0, in addptr units + int64_t unitBytes; // bytes advanced by one unit of stride }; // A unique key for grouping rewrites that can share an iter_arg. @@ -1439,12 +1503,14 @@ struct PostUpdateRewrite { // Keying on the original operands rather than on `initPtr` itself keeps the // comparison by Value identity meaningful: computeInitialPtr may materialize a // fresh pto.addptr per candidate, so equal start addresses do not necessarily -// share a Value. This is conservative — it can split groups that could have -// been merged — but never merges groups that must stay apart. -using IterArgGroupKey = std::tuple; +// share a Value. The effective byte unit is also part of the address sequence: +// equal numeric strides in element and byte ops need not advance equally. +// This is conservative — it can split groups that could have been merged — +// but never merges groups that must stay apart. +using IterArgGroupKey = std::tuple; static IterArgGroupKey getGroupKey(const PostUpdateRewrite &rw) { - return {rw.base, rw.strideOperand, rw.stride}; + return {rw.base, rw.strideOperand, rw.stride, rw.unitBytes}; } // Build the post-update form of an op while preserving every operand, @@ -1456,11 +1522,13 @@ static Operation *createPostUpdateOp(Operation *op, for (auto [i, operand] : llvm::enumerate(op->getOperands())) { if (static_cast(i) == info.baseOperandIdx) state.addOperands(base); - else if (static_cast(i) == info.strideOperandIdx) + else if (info.strideOperandIdx && i == *info.strideOperandIdx) state.addOperands(stride); else state.addOperands(operand); } + if (!info.strideOperandIdx) + state.addOperands(stride); state.addTypes(op->getResultTypes()); state.addTypes(base.getType()); state.addAttributes(op->getAttrs()); @@ -1476,7 +1544,7 @@ static Operation *createNormalOp(Operation *op, const PostUpdateOpInfo &info, for (auto [i, operand] : llvm::enumerate(op->getOperands())) { if (static_cast(i) == info.baseOperandIdx) state.addOperands(base); - else if (static_cast(i) == info.strideOperandIdx) + else if (info.strideOperandIdx && i == *info.strideOperandIdx) state.addOperands(zeroStride); else state.addOperands(operand); @@ -1621,10 +1689,11 @@ static scf::ForOp applyPostUpdateRewrites(scf::ForOp forOp, if (rewrites.empty()) return nullptr; - // Group rewrites by (base, stride). Ops in the same group share one iter_arg - // and all use the pre-update pointer. Only one updated_base per group is - // yielded. This avoids redundant iter_args for same-address ops (e.g. vlds - // + vsts both accessing %base[%iv]). + // Group rewrites by start-address operands, stride, and effective byte unit. + // Ops in the same group share one iter_arg and all use the pre-update + // pointer. Only one updated_base per group is yielded. This avoids redundant + // iter_args for same-address ops (e.g. vlds + vsts both accessing + // %base[%iv]) without merging byte- and element-scaled recurrences. DenseMap groupToIdx; // group key -> iter_arg index SmallVector rwGroupIdx(rewrites.size()); // rewrite -> group index SmallVector @@ -1738,6 +1807,8 @@ using SequentialExprCache = DenseMap; // SSA value is reused without guessing at its semantics. static StrideExprRef buildSequentialExpr(Value value, SequentialExprCache &cache) { + if (!value) + return makeConst(0); if (auto constant = getConstantIntValue(value)) return makeConst(*constant); if (auto it = cache.find(value); it != cache.end()) @@ -1847,16 +1918,28 @@ static bool validateSequentialRun(SequentialRun &run, return false; SequentialCandidate *first = run.candidates.front(); - run.strideType = first->strideOperand.getType(); + run.strideType = first->strideOperand + ? first->strideOperand.getType() + : IndexType::get(first->op->getContext()); + Value initialOffsetOperand = first->info->strideIsInitialOffset + ? first->strideOperand + : Value(); if (!canMaterializeAs(run.step, run.strideType) || !constantsFitType(run.step, run.strideType) || - !canScaleInitialOffset(first->strideOperand, first->elemBytes, + !satisfiesStrideConstraint(run.step, + first->info->strideConstraint) || + !canScaleInitialOffset(initialOffsetOperand, first->elemBytes, first->unitBytes)) return false; - for (SequentialCandidate *candidate : run.candidates) - if (candidate->strideOperand.getType() != run.strideType) + for (SequentialCandidate *candidate : run.candidates) { + Type candidateStrideType = + candidate->strideOperand + ? candidate->strideOperand.getType() + : IndexType::get(candidate->op->getContext()); + if (candidateStrideType != run.strideType) return false; + } SmallVector leaves; collectLeaves(run.step, leaves); @@ -1898,16 +1981,19 @@ static unsigned countDeadDynamicAddPtrs(const SequentialRun &run) { } static unsigned initialPointerCost(const SequentialRun &run) { - auto initialOffset = - getConstantIntValue(run.candidates.front()->strideOperand); + SequentialCandidate *first = run.candidates.front(); + if (!first->info->strideIsInitialOffset || !first->strideOperand) + return 0; + auto initialOffset = getConstantIntValue(first->strideOperand); return initialOffset && *initialOffset == 0 ? 0 : 1; } static bool isRunStrideUse(OpOperand &use, const SequentialRun &run) { return llvm::any_of(run.candidates, [&](SequentialCandidate *candidate) { - return use.getOwner() == candidate->op && + return candidate->info->strideOperandIdx && + use.getOwner() == candidate->op && use.getOperandNumber() == - static_cast(candidate->info->strideOperandIdx); + *candidate->info->strideOperandIdx; }); } @@ -1937,8 +2023,10 @@ static bool allUsesDisappearAfterRewrite(Operation *op, static bool cumulativeOffsetChainDefinitelyDies( const SequentialRun &run, DenseSet &deadOps) { for (SequentialCandidate *candidate : - llvm::drop_begin(run.candidates, 2)) - collectCumulativeOffsetOps(candidate->strideOperand, deadOps); + llvm::drop_begin(run.candidates, 2)) { + if (candidate->strideOperand) + collectCumulativeOffsetOps(candidate->strideOperand, deadOps); + } return !deadOps.empty() && llvm::all_of(deadOps, [&](Operation *op) { return allUsesDisappearAfterRewrite(op, deadOps, run); @@ -2011,8 +2099,10 @@ static bool isProfitableDirectSymbolicLeafRun( return candidate->base == candidate->rootBase; })) return false; - auto firstOffset = - getConstantIntValue(run.candidates.front()->strideOperand); + Value firstStrideOperand = run.candidates.front()->strideOperand; + auto firstOffset = firstStrideOperand + ? getConstantIntValue(firstStrideOperand) + : std::optional(0); if (!firstOffset || *firstOffset != 0) return false; @@ -2064,7 +2154,9 @@ static void processSequentialBlock(Block *block, DominanceInfo &dominance, auto elemBytes = addPtrUnitBytes(base); if (!elemBytes) continue; - int64_t unitBytes = strideUnitBytes(info->strideUnit, *elemBytes); + auto unitBytes = strideUnitBytes(&op, info->strideUnit, *elemBytes); + if (!unitBytes) + continue; NormalizedBase normalized = normalizeSequentialBase(base, *elemBytes, exprCache); @@ -2078,7 +2170,8 @@ static void processSequentialBlock(Block *block, DominanceInfo &dominance, } bucketIt->candidates.push_back( {&op, info, base, strideOperand, normalized.root, normalized.offset, - buildSequentialExpr(strideOperand, exprCache), *elemBytes, unitBytes}); + buildSequentialExpr(strideOperand, exprCache), *elemBytes, + *unitBytes}); } SmallVector runs; @@ -2136,8 +2229,11 @@ static void processSequentialBlock(Block *block, DominanceInfo &dominance, run.zeroStride = materializeSequential(makeConst(0), run.strideType, first->op->getLoc(), builder); builder.setInsertionPoint(first->op); + Value initialOffsetOperand = first->info->strideIsInitialOffset + ? first->strideOperand + : Value(); run.currentPtr = createInitialPtr( - first->base, first->strideOperand, first->info->strideUnit, + first->base, initialOffsetOperand, first->info->strideUnit, first->elemBytes, first->unitBytes, first->op->getLoc(), builder); } @@ -2236,20 +2332,24 @@ struct VPTOSoftPostUpdatePass std::optional elemBytes = addPtrUnitBytes(base); if (!elemBytes) continue; - int64_t unitBytes = strideUnitBytes(info->strideUnit, *elemBytes); + auto unitBytes = strideUnitBytes(&op, info->strideUnit, *elemBytes); + if (!unitBytes) + continue; // Analyze each operand independently: accumulator (iter_arg) first, // delta (IV/affine) fallback. Both return a symbolic per-iteration // stride; no IR is created until the candidate is known to be viable. DeltaCache deltaCache; StrideExprRef deltaBase = getStride(base, forOp, deltaCache); - StrideExprRef deltaOffset = getStride(strideOperand, forOp, deltaCache); + StrideExprRef deltaOffset = + strideOperand ? getStride(strideOperand, forOp, deltaCache) + : makeConst(0); if (!deltaBase || !deltaOffset) continue; StrideExprRef total = - combineStride(deltaBase, deltaOffset, *elemBytes, unitBytes); + combineStride(deltaBase, deltaOffset, *elemBytes, *unitBytes); if (!total) continue; @@ -2259,13 +2359,16 @@ struct VPTOSoftPostUpdatePass Type exprResultType; if (!exprType(total, exprResultType)) continue; - Type strideType = strideOperand.getType(); + Type strideType = + strideOperand ? strideOperand.getType() : builder.getIndexType(); if (exprResultType && exprResultType != strideType) continue; // Reject strides whose constants do not fit the target operand type. if (!constantsFitType(total, strideType)) continue; + if (!satisfiesStrideConstraint(total, info->strideConstraint)) + continue; // A stride built only from loop-invariant leaves is materialized before // the loop; otherwise it goes immediately before the candidate op. @@ -2291,12 +2394,16 @@ struct VPTOSoftPostUpdatePass Value strideNew = materialize(finalExpr, strideType, op.getLoc(), forOp, constCache, builder); - Value initPtr = computeInitialPtr(base, strideOperand, info->strideUnit, - *elemBytes, unitBytes, forOp, builder); + Value initialOffsetOperand = + info->strideIsInitialOffset ? strideOperand : Value(); + Value initPtr = computeInitialPtr( + base, initialOffsetOperand, info->strideUnit, *elemBytes, *unitBytes, + forOp, builder); if (!initPtr) continue; - rewrites.push_back({&op, base, strideOperand, strideNew, initPtr}); + rewrites.push_back( + {&op, base, strideOperand, strideNew, initPtr, *unitBytes}); } if (!rewrites.empty()) diff --git a/ptodsl/ptodsl/_ops.py b/ptodsl/ptodsl/_ops.py index 365ec64b04..f4116fe9f8 100644 --- a/ptodsl/ptodsl/_ops.py +++ b/ptodsl/ptodsl/_ops.py @@ -497,6 +497,7 @@ def vldus(source, align): op = _pto.VldusOp( result_type, _pto.AlignType.get(), + None, unwrap_surface_value(source), unwrap_surface_value(align), ) @@ -538,6 +539,7 @@ def vldsx2(source, offset_or_dist, dist=None, *, result_vreg_type=None): op = _pto.Vldsx2Op( result_type, result_type, + None, unwrap_surface_value(source), source_offset, _normalize_dist_token( @@ -557,6 +559,7 @@ def vldsx2(source, offset_or_dist, dist=None, *, result_vreg_type=None): op = _pto.Vldsx2Op( result_type, result_type, + None, unwrap_surface_value(source), _coerce_index(offset_or_dist, context="vldsx2(ptr, offset, dist)"), _normalize_dist_token( @@ -1159,6 +1162,7 @@ def vsldb(source, block_stride, repeat_stride, mask): return wrap_surface_value( _pto.VsldbOp( result_type, + None, unwrap_surface_value(source), _coerce_i16(block_stride, context="vsldb(..., block_stride, repeat_stride, mask)"), _coerce_i16(repeat_stride, context="vsldb(..., block_stride, repeat_stride, mask)"), @@ -1697,6 +1701,7 @@ def plds(buf, offset, *, dist="NORM"): return wrap_surface_value( _pto.PldsOp( result_type, + None, unwrap_surface_value(buf), _coerce_index(offset, context="plds(buf, offset)"), _normalize_predicate_dist( @@ -1712,6 +1717,7 @@ def psts(mask_value, buf, offset, *, dist="NORM"): """``pto.psts`` – store a predicate mask to UB memory.""" _infer_mask_metadata(mask_value, context="psts(mask, buf, offset)") _pto.PstsOp( + None, unwrap_surface_value(mask_value), unwrap_surface_value(buf), _coerce_index(offset, context="psts(mask, buf, offset)"), @@ -1759,6 +1765,7 @@ def vstar(align, destination): def vstas(align, destination, offset): """``pto.vstas`` – flush alignment-buffered tail bytes with an explicit offset.""" _pto.VstasOp( + None, unwrap_surface_value(align), unwrap_surface_value(destination), _coerce_i32(offset, context="vstas(align, destination, offset)"), @@ -1783,6 +1790,7 @@ def vstus(align_in, offset, value, base): return wrap_surface_value( _pto.VstusOp( _pto.AlignType.get(), + None, unwrap_surface_value(align_in), _coerce_i32(offset, context="vstus(align, offset, value, base)"), unwrap_surface_value(value), diff --git a/test/lit/vpto/issue_173_vpto_llvm.pto b/test/lit/vpto/issue_173_vpto_llvm.pto index b623226394..5d7725b866 100644 --- a/test/lit/vpto/issue_173_vpto_llvm.pto +++ b/test/lit/vpto/issue_173_vpto_llvm.pto @@ -31,6 +31,6 @@ module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind&1 | FileCheck %s +// RUN: ptoas --cann-output-version=9.0.0-beta.1 --pto-arch=a5 --pto-backend=vpto --emit-vpto-llvm-ir %s -o - 2>&1 | FileCheck %s + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @canonical_integer_memory( + %src: !pto.ptr, %dst: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %value, %next_src = pto.vlds %src[%c0] + : !pto.ptr -> !pto.vreg<64xsi32>, !pto.ptr + %next_dst = pto.vsts %value, %dst[%c0], %mask {dist = "NORM_B32"} + : !pto.vreg<64xsi32>, !pto.ptr, !pto.mask + -> !pto.ptr + pto.vsts %value, %next_dst[%c0], %mask {dist = "NORM_B32"} + : !pto.vreg<64xsi32>, !pto.ptr, !pto.mask + } + return + } + + func.func @canonical_x2_f16( + %src: !pto.ptr, %dst: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + pto.vecscope { + %mask = pto.pset_b16 "PAT_ALL" : !pto.mask + %low, %high = pto.vldsx2 %src[%c0], "DINTLV_B16" + : !pto.ptr, index + -> !pto.vreg<128xf16>, !pto.vreg<128xf16> + pto.vstsx2 %low, %high, %dst[%c0], "INTLV_B16", %mask + : !pto.vreg<128xf16>, !pto.vreg<128xf16>, + !pto.ptr, index, !pto.mask + } + return + } + + func.func @canonical_vsldb_ui64( + %src: !pto.ptr, %dst: !pto.ptr) attributes {pto.kernel} { + %offset = arith.constant 0 : index + %c0 = arith.constant 0 : i16 + %c1 = arith.constant 1 : i16 + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %value = pto.vsldb %src, %c1, %c0, %mask + : !pto.ptr, i16, i16, !pto.mask + -> !pto.vreg<32xui64> + pto.vsts %value, %dst[%offset], %mask {dist = "PK_B64"} + : !pto.vreg<32xui64>, !pto.ptr, !pto.mask + } + return + } + + func.func @canonical_fp8_block_post( + %value: !pto.vreg<256xf8E4M3FN>, + %dst: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : i16 + %c1 = arith.constant 1 : i16 + pto.vecscope { + %mask = pto.pset_b8 "PAT_ALL" : !pto.mask + %next = pto.vsstb %value, %dst, %c1, %c0, %mask + : !pto.vreg<256xf8E4M3FN>, !pto.ptr, + i16, i16, !pto.mask -> !pto.ptr + pto.vsstb %value, %next, %c1, %c0, %mask + : !pto.vreg<256xf8E4M3FN>, !pto.ptr, + i16, i16, !pto.mask + } + return + } +} + +// CHECK-LABEL: define void @canonical_integer_memory_mix_aiv +// CHECK: call { <64 x i32>, ptr addrspace(6) } @llvm.hivm.vldsx1.post.v64i32 +// CHECK: call ptr addrspace(6) @llvm.hivm.vstsx1.post.v64i32 +// CHECK: call void @llvm.hivm.vstsx1.v64i32 + +// CHECK-LABEL: define void @canonical_x2_f16_mix_aiv +// CHECK: call { <128 x half>, <128 x half> } @llvm.hivm.vldsx2.v128f16 +// CHECK: call void @llvm.hivm.vstsx2.v128f16(<128 x half> + +// CHECK-LABEL: define void @canonical_vsldb_ui64_mix_aiv +// CHECK: call <32 x i64> @llvm.hivm.vsldb.v32i64 + +// CHECK-LABEL: define void @canonical_fp8_block_post_mix_aiv +// CHECK: call ptr addrspace(6) @llvm.hivm.vsstb.post.v256i8 +// CHECK: call void @llvm.hivm.vsstb.v256i8 diff --git a/test/lit/vpto/predicate_post_update_result_vpto_llvm.pto b/test/lit/vpto/predicate_post_update_result_vpto_llvm.pto new file mode 100644 index 0000000000..23467349f0 --- /dev/null +++ b/test/lit/vpto/predicate_post_update_result_vpto_llvm.pto @@ -0,0 +1,41 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software; you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: ptoas --cann-output-version=9.0.0 --pto-arch=a5 --pto-backend=vpto --emit-vpto-llvm-ir %s -o - 2>&1 | FileCheck %s +// RUN: ptoas --cann-output-version=9.0.0-beta.1 --pto-arch=a5 --pto-backend=vpto --emit-vpto-llvm-ir %s -o - 2>&1 | FileCheck %s + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @predicate_post_update( + %src: !pto.ptr, %dst: !pto.ptr) attributes {pto.kernel} { + %c4 = arith.constant 4 : index + pto.vecscope { + %dynamic_mask, %next_dynamic_src = pto.plds %src[%c4], "DS" + : !pto.ptr, index -> !pto.mask, !pto.ptr + %immediate_mask, %next_immediate_src = pto.pldi %next_dynamic_src[%c4], "US" + : !pto.ptr, index -> !pto.mask, !pto.ptr + %next_dynamic_dst = pto.psts %dynamic_mask, %dst[%c4], "PK" + : !pto.mask, !pto.ptr, index -> !pto.ptr + %next_immediate_dst = pto.psti %immediate_mask, %next_dynamic_dst[%c4], "NORM" + : !pto.mask, !pto.ptr, index -> !pto.ptr + } + return + } +} + +// CHECK-DAG: declare { <256 x i1>, ptr addrspace(6) } @llvm.hivm.plds.post.b8 +// CHECK-DAG: declare { <256 x i1>, ptr addrspace(6) } @llvm.hivm.pldi.post.b8 +// CHECK-DAG: declare ptr addrspace(6) @llvm.hivm.psts.post.b8 +// CHECK-DAG: declare ptr addrspace(6) @llvm.hivm.psti.post.b8 +// CHECK: call { <256 x i1>, ptr addrspace(6) } @llvm.hivm.plds.post.b8 +// CHECK-SAME: i32 2, i32 1) +// CHECK: call { <256 x i1>, ptr addrspace(6) } @llvm.hivm.pldi.post.b8 +// CHECK-SAME: i32 1, i32 1) +// CHECK: call ptr addrspace(6) @llvm.hivm.psts.post.b8 +// CHECK-SAME: i32 1, i32 1) +// CHECK: call ptr addrspace(6) @llvm.hivm.psti.post.b8 +// CHECK-SAME: i32 0, i32 1) diff --git a/test/lit/vpto/predicate_post_update_verify_invalid.pto b/test/lit/vpto/predicate_post_update_verify_invalid.pto new file mode 100644 index 0000000000..ecb9e26d1e --- /dev/null +++ b/test/lit/vpto/predicate_post_update_verify_invalid.pto @@ -0,0 +1,69 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software; you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: not pto-test-opt %s -split-input-file 2>&1 | FileCheck %s + +module { + func.func @plds_requires_matching_updated_base(%src: !pto.ptr) { + %c0 = arith.constant 0 : index + pto.vecscope { + %mask, %next = pto.plds %src[%c0], "NORM" + : !pto.ptr, index -> !pto.mask, !pto.ptr + } + return + } +} + +// CHECK: 'pto.plds' op requires updated base result to match base type + +// ----- + +module { + func.func @pldi_requires_matching_updated_base(%src: !pto.ptr) { + %c0 = arith.constant 0 : index + pto.vecscope { + %mask, %next = pto.pldi %src[%c0], "NORM" + : !pto.ptr, index -> !pto.mask, !pto.ptr + } + return + } +} + +// CHECK: 'pto.pldi' op requires updated base result to match base type + +// ----- + +module { + func.func @psts_requires_matching_updated_base( + %mask: !pto.mask, %dst: !pto.ptr) { + %c0 = arith.constant 0 : index + pto.vecscope { + %next = pto.psts %mask, %dst[%c0], "NORM" + : !pto.mask, !pto.ptr, index -> !pto.ptr + } + return + } +} + +// CHECK: 'pto.psts' op requires updated base result to match base type + +// ----- + +module { + func.func @psti_requires_matching_updated_base( + %mask: !pto.mask, %dst: !pto.ptr) { + %c0 = arith.constant 0 : index + pto.vecscope { + %next = pto.psti %mask, %dst[%c0], "NORM" + : !pto.mask, !pto.ptr, index -> !pto.ptr + } + return + } +} + +// CHECK: 'pto.psti' op requires updated base result to match base type diff --git a/test/lit/vpto/soft_postupdate_address_unit_regressions.pto b/test/lit/vpto/soft_postupdate_address_unit_regressions.pto new file mode 100644 index 0000000000..abc1349ca0 --- /dev/null +++ b/test/lit/vpto/soft_postupdate_address_unit_regressions.pto @@ -0,0 +1,97 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software; you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: ptoas --pto-backend=vpto --pto-arch=a5 --enable-vpto-soft-postupdate --emit-vpto %s -o - | FileCheck %s +// RUN: ptoas --cann-output-version=9.0.0 --pto-backend=vpto --pto-arch=a5 --enable-vpto-soft-postupdate --emit-vpto-llvm-ir %s -o - 2>&1 | FileCheck %s --check-prefix=LLVM +// RUN: ptoas --cann-output-version=9.0.0-beta.1 --pto-backend=vpto --pto-arch=a5 --enable-vpto-soft-postupdate --emit-vpto-llvm-ir %s -o - 2>&1 | FileCheck %s --check-prefix=LLVM + +// Element and byte strides with the same numeric SSA value are different +// address recurrences on an f32 pointer and must use separate iter_args. +// CHECK-LABEL: func.func @mixed_element_and_byte_units +// CHECK: scf.for {{.*}} iter_args([[ELEMENT_PTR:%[a-zA-Z0-9_]+]] = %arg0, [[BYTE_PTR:%[a-zA-Z0-9_]+]] = %arg0) -> (!pto.ptr, !pto.ptr) +// CHECK: [[VALUE:%[a-zA-Z0-9_]+]], [[ELEMENT_NEXT:%[a-zA-Z0-9_]+]] = pto.vlds [[ELEMENT_PTR]][%c1] +// CHECK: pto.vsts [[VALUE]], %arg1[%c0] +// CHECK: [[BYTE_NEXT:%[a-zA-Z0-9_]+]] = pto.psts {{.*}}, [[BYTE_PTR]][%c1] +// CHECK: scf.yield [[ELEMENT_NEXT]], [[BYTE_NEXT]] + +// Nonzero i32 initial offsets must be converted to index before pto.addptr. +// CHECK-LABEL: func.func @vstas_nonzero_i32_initial_offset +// CHECK: %[[VSTAS_INIT:[a-zA-Z0-9_]+]] = pto.addptr %arg0, %c4 +// CHECK: scf.for {{.*}} iter_args({{%.*}} = %[[VSTAS_INIT]]) +// CHECK: pto.vstas {{.*}}, {{%.*}}, %c4_i32 +// LLVM-LABEL: define void @vstas_nonzero_i32_initial_offset_mix_aiv +// LLVM: call ptr addrspace(6) @llvm.hivm.vstas.post(<32 x i8> {{.*}}, ptr addrspace(6) {{.*}}, i32 16, i32 1) + +// sprsti offsets are signed 8-bit word counts; -1 must remain negative when +// the initial pto.addptr address is constructed. +// CHECK-LABEL: func.func @sprsti_negative_initial_offset +// CHECK: %c-1 = arith.constant -1 : index +// CHECK: %[[SPR_INIT:[a-zA-Z0-9_]+]] = pto.addptr %arg0, %c-1 +// CHECK: scf.for {{.*}} iter_args({{%.*}} = %[[SPR_INIT]]) +// CHECK: pto.sprsti "AR", {{%.*}}[%c1_i32] + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @mixed_element_and_byte_units( + %base: !pto.ptr, + %sink: !pto.ptr, + %predicate_mask: !pto.mask, + %vector_mask: !pto.mask) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c4 = arith.constant 4 : index + pto.vecscope { + scf.for %iv = %c0 to %c4 step %c1 { + %value = pto.vlds %base[%iv] + : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %value, %sink[%c0], %vector_mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + pto.psts %predicate_mask, %base[%iv], "NORM" + : !pto.mask, !pto.ptr, index + } + } + return + } + + func.func @vstas_nonzero_i32_initial_offset( + %base: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c4 = arith.constant 4 : index + %c4_i32 = arith.constant 4 : i32 + pto.vecscope { + scf.for %iv = %c0 to %c4 step %c1 + iter_args(%offset = %c4_i32) -> (i32) { + %align = pto.init_align : !pto.align + pto.vstas %align, %base, %offset + : !pto.align, !pto.ptr, i32 + %next_offset = arith.addi %offset, %c4_i32 : i32 + scf.yield %next_offset : i32 + } + } + return + } + + func.func @sprsti_negative_initial_offset( + %base: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c4 = arith.constant 4 : index + %cm1_i32 = arith.constant -1 : i32 + pto.vecscope { + scf.for %iv = %c0 to %c4 step %c1 + iter_args(%ptr = %base) -> (!pto.ptr) { + pto.sprsti "AR", %ptr[%cm1_i32] + : !pto.ptr, i32 + %next_ptr = pto.addptr %ptr, %c1 + : !pto.ptr -> !pto.ptr + scf.yield %next_ptr : !pto.ptr + } + } + return + } +} diff --git a/test/lit/vpto/soft_postupdate_predicate_memory.pto b/test/lit/vpto/soft_postupdate_predicate_memory.pto new file mode 100644 index 0000000000..b7ab21e350 --- /dev/null +++ b/test/lit/vpto/soft_postupdate_predicate_memory.pto @@ -0,0 +1,195 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software; you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: ptoas --pto-backend=vpto --pto-arch=a5 --enable-vpto-soft-postupdate --emit-vpto %s -o - | FileCheck %s + +// CHECK-LABEL: func.func @sequential_plds +// CHECK: %{{.*}}, [[PLDS1:%[a-zA-Z0-9_]+]] = pto.plds {{%.*}}[[[PLDS_STEP:%[a-zA-Z0-9_]+]]] +// CHECK: %{{.*}}, [[PLDS2:%[a-zA-Z0-9_]+]] = pto.plds [[PLDS1]][[[PLDS_STEP]]] +// CHECK: %{{.*}} = pto.plds [[PLDS2]][%c0] + +// CHECK-LABEL: func.func @loop_pldi +// CHECK: scf.for +// CHECK: %{{.*}}, %{{.*}} = pto.pldi {{%.*}}[%c4] + +// CHECK-LABEL: func.func @sequential_psts +// CHECK: [[PSTS1:%[a-zA-Z0-9_]+]] = pto.psts {{.*}}[[[PSTS_STEP:%[a-zA-Z0-9_]+]]] +// CHECK: [[PSTS2:%[a-zA-Z0-9_]+]] = pto.psts {{.*}}[[PSTS1]][[[PSTS_STEP]]] +// CHECK: pto.psts {{.*}}[[PSTS2]][%c0] + +// CHECK-LABEL: func.func @loop_psti +// CHECK: scf.for +// CHECK: %{{.*}} = pto.psti {{.*}}[%c4] + +// Immediate predicate offsets count dist-dependent alignment units. Each +// original offset and addptr step below is one alignment unit. The pass must +// build a correctly scaled initial pointer and emit post-update stride 1. +// CHECK-LABEL: func.func @immediate_base_steps_use_alignment_units +// CHECK: [[PLDI_NORM_INIT:%[a-zA-Z0-9_]+]] = pto.addptr {{%.*}}, %c32 +// CHECK-NEXT: [[PLDI_US_INIT:%[a-zA-Z0-9_]+]] = pto.addptr {{%.*}}, %c16 +// CHECK-NEXT: [[PLDI_DS_INIT:%[a-zA-Z0-9_]+]] = pto.addptr {{%.*}}, %c32 +// CHECK-NEXT: [[PSTI_NORM_INIT:%[a-zA-Z0-9_]+]] = pto.addptr {{%.*}}, %c32 +// CHECK-NEXT: [[PSTI_PK_INIT:%[a-zA-Z0-9_]+]] = pto.addptr {{%.*}}, %c16 +// CHECK: scf.for +// CHECK-SAME: iter_args({{%.* = %.*}}, [[PLDI_NORM_BASE:%[a-zA-Z0-9_]+]] = [[PLDI_NORM_INIT]], [[PLDI_US_BASE:%[a-zA-Z0-9_]+]] = [[PLDI_US_INIT]], [[PLDI_DS_BASE:%[a-zA-Z0-9_]+]] = [[PLDI_DS_INIT]], [[PSTI_NORM_BASE:%[a-zA-Z0-9_]+]] = [[PSTI_NORM_INIT]], [[PSTI_PK_BASE:%[a-zA-Z0-9_]+]] = [[PSTI_PK_INIT]]) +// CHECK: [[MASK_NORM:%[a-zA-Z0-9_]+]], [[PLDI_NORM_NEXT:%[a-zA-Z0-9_]+]] = pto.pldi [[PLDI_NORM_BASE]][%c1], "NORM" +// CHECK-NEXT: [[MASK_US:%[a-zA-Z0-9_]+]], [[PLDI_US_NEXT:%[a-zA-Z0-9_]+]] = pto.pldi [[PLDI_US_BASE]][%c1], "US" +// CHECK-NEXT: [[MASK_DS:%[a-zA-Z0-9_]+]], [[PLDI_DS_NEXT:%[a-zA-Z0-9_]+]] = pto.pldi [[PLDI_DS_BASE]][%c1], "DS" +// CHECK-NEXT: [[PSTI_NORM_NEXT:%[a-zA-Z0-9_]+]] = pto.psti [[MASK_NORM]], [[PSTI_NORM_BASE]][%c1], "NORM" +// CHECK-NEXT: [[PSTI_PK_NEXT:%[a-zA-Z0-9_]+]] = pto.psti [[MASK_US]], [[PSTI_PK_BASE]][%c1], "PK" +// CHECK: scf.yield {{%.*}}, [[PLDI_NORM_NEXT]], [[PLDI_US_NEXT]], [[PLDI_DS_NEXT]], [[PSTI_NORM_NEXT]], [[PSTI_PK_NEXT]] + +// Dynamic strides are legal for plds/psts but not for the immediate pldi/psti +// forms. The latter remain unchanged when their base step is symbolic. +// CHECK-LABEL: func.func @dynamic_immediate_stride_is_skipped +// CHECK: scf.for +// CHECK: {{^ *}}%{{[^, ]+}} = pto.pldi +// CHECK: {{^ *}}pto.psti + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @sequential_plds( + %src: !pto.ptr, + %sink: !pto.ptr, + %step: index) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %step2 = arith.addi %step, %step : index + pto.vecscope { + %m0 = pto.plds %src[%c0], "NORM" : !pto.ptr, index -> !pto.mask + pto.psti %m0, %sink[%c0], "NORM" : !pto.mask, !pto.ptr, index + %m1 = pto.plds %src[%step], "NORM" : !pto.ptr, index -> !pto.mask + pto.psti %m1, %sink[%c0], "NORM" : !pto.mask, !pto.ptr, index + %m2 = pto.plds %src[%step2], "NORM" : !pto.ptr, index -> !pto.mask + pto.psti %m2, %sink[%c0], "NORM" : !pto.mask, !pto.ptr, index + } + return + } + + func.func @loop_pldi(%src: !pto.ptr, %sink: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : index + %c128 = arith.constant 128 : index + pto.vecscope { + scf.for %iv = %c0 to %c3 step %c1 + iter_args(%src_ptr = %src) -> (!pto.ptr) { + %mask = pto.pldi %src_ptr[%c0], "NORM" + : !pto.ptr, index -> !pto.mask + pto.psti %mask, %sink[%c0], "NORM" + : !pto.mask, !pto.ptr, index + %next_src = pto.addptr %src_ptr, %c128 + : !pto.ptr -> !pto.ptr + scf.yield %next_src : !pto.ptr + } + } + return + } + + func.func @sequential_psts( + %mask: !pto.mask, + %dst: !pto.ptr, + %step: index) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %step2 = arith.addi %step, %step : index + pto.vecscope { + pto.psts %mask, %dst[%c0], "NORM" : !pto.mask, !pto.ptr, index + pto.psts %mask, %dst[%step], "NORM" : !pto.mask, !pto.ptr, index + pto.psts %mask, %dst[%step2], "NORM" : !pto.mask, !pto.ptr, index + } + return + } + + func.func @loop_psti(%mask: !pto.mask, %dst: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : index + %c128 = arith.constant 128 : index + pto.vecscope { + scf.for %iv = %c0 to %c3 step %c1 + iter_args(%dst_ptr = %dst) -> (!pto.ptr) { + pto.psti %mask, %dst_ptr[%c0], "NORM" + : !pto.mask, !pto.ptr, index + %next_dst = pto.addptr %dst_ptr, %c128 + : !pto.ptr -> !pto.ptr + scf.yield %next_dst : !pto.ptr + } + } + return + } + + func.func @immediate_base_steps_use_alignment_units( + %src_norm: !pto.ptr, + %src_us: !pto.ptr, + %src_ds: !pto.ptr, + %dst_norm: !pto.ptr, + %dst_pk: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : index + %c16 = arith.constant 16 : index + %c32 = arith.constant 32 : index + pto.vecscope { + scf.for %iv = %c0 to %c3 step %c1 + iter_args(%src_norm_ptr = %src_norm, + %src_us_ptr = %src_us, + %src_ds_ptr = %src_ds, + %dst_norm_ptr = %dst_norm, + %dst_pk_ptr = %dst_pk) + -> (!pto.ptr, !pto.ptr, !pto.ptr, + !pto.ptr, !pto.ptr) { + %mask_norm = pto.pldi %src_norm_ptr[%c1], "NORM" + : !pto.ptr, index -> !pto.mask + %mask_us = pto.pldi %src_us_ptr[%c1], "US" + : !pto.ptr, index -> !pto.mask + %mask_ds = pto.pldi %src_ds_ptr[%c1], "DS" + : !pto.ptr, index -> !pto.mask + pto.psti %mask_norm, %dst_norm_ptr[%c1], "NORM" + : !pto.mask, !pto.ptr, index + pto.psti %mask_us, %dst_pk_ptr[%c1], "PK" + : !pto.mask, !pto.ptr, index + // Keep the DS load observable. This NORM store intentionally cannot + // follow the 16-byte dst_pk recurrence and remains in normal form. + pto.psti %mask_ds, %dst_pk_ptr[%c0], "NORM" + : !pto.mask, !pto.ptr, index + %next_src_norm = pto.addptr %src_norm_ptr, %c32 + : !pto.ptr -> !pto.ptr + %next_src_us = pto.addptr %src_us_ptr, %c16 + : !pto.ptr -> !pto.ptr + %next_src_ds = pto.addptr %src_ds_ptr, %c32 + : !pto.ptr -> !pto.ptr + %next_dst_norm = pto.addptr %dst_norm_ptr, %c32 + : !pto.ptr -> !pto.ptr + %next_dst_pk = pto.addptr %dst_pk_ptr, %c16 + : !pto.ptr -> !pto.ptr + scf.yield %next_src_norm, %next_src_us, %next_src_ds, + %next_dst_norm, %next_dst_pk + : !pto.ptr, !pto.ptr, !pto.ptr, + !pto.ptr, !pto.ptr + } + } + return + } + + func.func @dynamic_immediate_stride_is_skipped( + %src: !pto.ptr, %dst: !pto.ptr, %step: index) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c4 = arith.constant 4 : index + pto.vecscope { + scf.for %iv = %c0 to %c4 step %c1 { + %scaled = arith.muli %iv, %step : index + %src_at = pto.addptr %src, %scaled : !pto.ptr -> !pto.ptr + %dst_at = pto.addptr %dst, %scaled : !pto.ptr -> !pto.ptr + %mask = pto.pldi %src_at[%c0], "NORM" + : !pto.ptr, index -> !pto.mask + pto.psti %mask, %dst_at[%c0], "NORM" + : !pto.mask, !pto.ptr, index + } + } + return + } +} diff --git a/test/lit/vpto/soft_postupdate_spr_vstas.pto b/test/lit/vpto/soft_postupdate_spr_vstas.pto new file mode 100644 index 0000000000..7faa0747a6 --- /dev/null +++ b/test/lit/vpto/soft_postupdate_spr_vstas.pto @@ -0,0 +1,118 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software; you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: ptoas --pto-backend=vpto --pto-arch=a5 --enable-vpto-soft-postupdate --emit-vpto %s -o - | FileCheck %s + +// CHECK-LABEL: func.func @loop_sprsti +// CHECK: scf.for +// CHECK: %{{.*}} = pto.sprsti "AR", {{%.*}}[%c4{{.*}}] + +// CHECK-LABEL: func.func @sequential_sprsts +// CHECK: [[SPRSTS1:%[a-zA-Z0-9_]+]] = pto.sprsts "AR", {{%.*}}[[[SPRSTS_STEP:%[a-zA-Z0-9_]+]]] +// CHECK: [[SPRSTS2:%[a-zA-Z0-9_]+]] = pto.sprsts "AR", [[SPRSTS1]][[[SPRSTS_STEP]]] +// CHECK: pto.sprsts "AR", [[SPRSTS2]][%c0{{.*}}] + +// sprsti immediate offsets count 4-byte words. A one-element ui32 addptr +// step must become post-update stride 1. +// CHECK-LABEL: func.func @sprsti_base_step_uses_words +// CHECK: scf.for +// CHECK: %{{.*}} = pto.sprsti "AR", {{%.*}}[%c1{{.*}}] + +// CHECK-LABEL: func.func @sprsti_out_of_range_is_skipped +// CHECK: scf.for +// CHECK: pto.sprsti "AR", {{%.*}}[%c0{{.*}}] : !pto.ptr, i32 + +// CHECK-LABEL: func.func @sequential_vstas +// CHECK: [[VSTAS1:%[a-zA-Z0-9_]+]] = pto.vstas {{.*}}, {{%.*}}, [[VSTAS_STEP:%[a-zA-Z0-9_]+]] +// CHECK: [[VSTAS2:%[a-zA-Z0-9_]+]] = pto.vstas {{.*}}, [[VSTAS1]], [[VSTAS_STEP]] +// CHECK: pto.vstas {{.*}}, [[VSTAS2]], %c0{{.*}} + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @loop_sprsti(%dst: !pto.ptr) attributes {pto.kernel} { + %c0_index = arith.constant 0 : index + %c1_index = arith.constant 1 : index + %c3_index = arith.constant 3 : index + %c4_index = arith.constant 4 : index + %c0 = arith.constant 0 : i32 + pto.vecscope { + scf.for %iv = %c0_index to %c3_index step %c1_index + iter_args(%dst_ptr = %dst) -> (!pto.ptr) { + pto.sprsti "AR", %dst_ptr[%c0] : !pto.ptr, i32 + %next_dst = pto.addptr %dst_ptr, %c4_index + : !pto.ptr -> !pto.ptr + scf.yield %next_dst : !pto.ptr + } + } + return + } + + func.func @sequential_sprsts( + %dst: !pto.ptr, + %step: i32) attributes {pto.kernel} { + %c0 = arith.constant 0 : i32 + %step2 = arith.addi %step, %step : i32 + pto.vecscope { + pto.sprsts "AR", %dst[%c0] : !pto.ptr, i32 + pto.sprsts "AR", %dst[%step] : !pto.ptr, i32 + pto.sprsts "AR", %dst[%step2] : !pto.ptr, i32 + } + return + } + + func.func @sprsti_base_step_uses_words( + %dst: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : i32 + %c0_index = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : index + pto.vecscope { + scf.for %iv = %c0_index to %c3 step %c1 + iter_args(%dst_ptr = %dst) -> (!pto.ptr) { + pto.sprsti "AR", %dst_ptr[%c0] : !pto.ptr, i32 + %next_dst = pto.addptr %dst_ptr, %c1 + : !pto.ptr -> !pto.ptr + scf.yield %next_dst : !pto.ptr + } + } + return + } + + func.func @sprsti_out_of_range_is_skipped( + %dst: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : i32 + %c0_index = arith.constant 0 : index + %c1_index = arith.constant 1 : index + %c4_index = arith.constant 4 : index + %c128_index = arith.constant 128 : index + pto.vecscope { + scf.for %iv = %c0_index to %c4_index step %c1_index { + %scaled = arith.muli %iv, %c128_index : index + %dst_at = pto.addptr %dst, %scaled + : !pto.ptr -> !pto.ptr + pto.sprsti "AR", %dst_at[%c0] : !pto.ptr, i32 + } + } + return + } + + func.func @sequential_vstas( + %dst: !pto.ptr, + %step: i32) attributes {pto.kernel} { + %c0 = arith.constant 0 : i32 + %step2 = arith.addi %step, %step : i32 + pto.vecscope { + %align0 = pto.init_align : !pto.align + pto.vstas %align0, %dst, %c0 : !pto.align, !pto.ptr, i32 + %align1 = pto.init_align : !pto.align + pto.vstas %align1, %dst, %step : !pto.align, !pto.ptr, i32 + %align2 = pto.init_align : !pto.align + pto.vstas %align2, %dst, %step2 : !pto.align, !pto.ptr, i32 + } + return + } +} diff --git a/test/lit/vpto/soft_postupdate_target_alignment.pto b/test/lit/vpto/soft_postupdate_target_alignment.pto new file mode 100644 index 0000000000..f253cb2a9e --- /dev/null +++ b/test/lit/vpto/soft_postupdate_target_alignment.pto @@ -0,0 +1,45 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software; you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: pto-test-opt %s -vpto-soft-postupdate | FileCheck %s + +// A5 predicate alignment sizes must not be assumed for another target. +// CHECK-LABEL: func.func @a3_predicate_alignment_is_unknown +// CHECK: scf.for +// CHECK: %{{.*}} = pto.pldi %{{.*}}[%c0], "US" +// CHECK-NOT: pto.pldi {{.*}} -> !pto.mask, !pto.ptr +// CHECK: pto.psti %{{.*}}, %{{.*}}[%c0], "PK" +// CHECK-NOT: pto.psti {{.*}} -> !pto.ptr +// CHECK: pto.addptr + +module attributes {pto.target_arch = "a3", pto.kernel_kind = #pto.kernel_kind} { + func.func @a3_predicate_alignment_is_unknown( + %src: !pto.ptr, %dst: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c16 = arith.constant 16 : index + pto.vecscope { + scf.for %iv = %c0 to %c2 step %c1 + iter_args(%src_ptr = %src, %dst_ptr = %dst) + -> (!pto.ptr, !pto.ptr) { + %mask = pto.pldi %src_ptr[%c0], "US" + : !pto.ptr, index -> !pto.mask + pto.psti %mask, %dst_ptr[%c0], "PK" + : !pto.mask, !pto.ptr, index + %next_src = pto.addptr %src_ptr, %c16 + : !pto.ptr -> !pto.ptr + %next_dst = pto.addptr %dst_ptr, %c16 + : !pto.ptr -> !pto.ptr + scf.yield %next_src, %next_dst + : !pto.ptr, !pto.ptr + } + } + return + } +} diff --git a/test/lit/vpto/soft_postupdate_vldsx2_vsldb.pto b/test/lit/vpto/soft_postupdate_vldsx2_vsldb.pto new file mode 100644 index 0000000000..7ce0f8ead3 --- /dev/null +++ b/test/lit/vpto/soft_postupdate_vldsx2_vsldb.pto @@ -0,0 +1,77 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software; you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: ptoas --pto-backend=vpto --pto-arch=a5 --enable-vpto-soft-postupdate --emit-vpto %s -o - | FileCheck %s + +// CHECK-LABEL: func.func @sequential_vldsx2 +// CHECK: %{{.*}}, %{{.*}}, [[X1:%[a-zA-Z0-9_]+]] = pto.vldsx2 {{%.*}}[[[XSTEP:%[a-zA-Z0-9_]+]]] +// CHECK: %{{.*}}, %{{.*}}, [[X2:%[a-zA-Z0-9_]+]] = pto.vldsx2 [[X1]][[[XSTEP]]] +// CHECK: %{{.*}}, %{{.*}} = pto.vldsx2 [[X2]][%c0] + +// CHECK-LABEL: func.func @sequential_vsldb +// CHECK: %{{.*}}, [[B1:%[a-zA-Z0-9_]+]] = pto.vsldb {{%.*}}, %c2_i16, [[BSTEP:%[a-zA-Z0-9_]+]] +// CHECK: %{{.*}}, [[B2:%[a-zA-Z0-9_]+]] = pto.vsldb [[B1]], %c2_i16, [[BSTEP]] +// CHECK: %{{.*}} = pto.vsldb [[B2]], %c2_i16, %c0_i16 + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @sequential_vldsx2( + %src: !pto.ptr, + %sink: !pto.ptr, + %step: index) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %step2 = arith.addi %step, %step : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %l0, %h0 = pto.vldsx2 %src[%c0], "DINTLV_B32" + : !pto.ptr, index -> !pto.vreg<64xf32>, !pto.vreg<64xf32> + pto.vsts %l0, %sink[%c0], %mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + pto.vsts %h0, %sink[%c0], %mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %l1, %h1 = pto.vldsx2 %src[%step], "DINTLV_B32" + : !pto.ptr, index -> !pto.vreg<64xf32>, !pto.vreg<64xf32> + pto.vsts %l1, %sink[%c0], %mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + pto.vsts %h1, %sink[%c0], %mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %l2, %h2 = pto.vldsx2 %src[%step2], "DINTLV_B32" + : !pto.ptr, index -> !pto.vreg<64xf32>, !pto.vreg<64xf32> + pto.vsts %l2, %sink[%c0], %mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + pto.vsts %h2, %sink[%c0], %mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } + + func.func @sequential_vsldb( + %src: !pto.ptr, + %sink: !pto.ptr, + %step: i16) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c0_i16 = arith.constant 0 : i16 + %c2_i16 = arith.constant 2 : i16 + %step2 = arith.addi %step, %step : i16 + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %v0 = pto.vsldb %src, %c2_i16, %c0_i16, %mask + : !pto.ptr, i16, i16, !pto.mask -> !pto.vreg<64xf32> + pto.vsts %v0, %sink[%c0], %mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v1 = pto.vsldb %src, %c2_i16, %step, %mask + : !pto.ptr, i16, i16, !pto.mask -> !pto.vreg<64xf32> + pto.vsts %v1, %sink[%c0], %mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v2 = pto.vsldb %src, %c2_i16, %step2, %mask + : !pto.ptr, i16, i16, !pto.mask -> !pto.vreg<64xf32> + pto.vsts %v2, %sink[%c0], %mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } +} diff --git a/test/lit/vpto/soft_postupdate_vldus_vstus.pto b/test/lit/vpto/soft_postupdate_vldus_vstus.pto new file mode 100644 index 0000000000..78fcbc143e --- /dev/null +++ b/test/lit/vpto/soft_postupdate_vldus_vstus.pto @@ -0,0 +1,88 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: ptoas --pto-backend=vpto --pto-arch=a5 --enable-vpto-soft-postupdate --emit-vpto %s -o - | FileCheck %s + +// CHECK-LABEL: func.func @loop_vldus_vstus +// vstus offset is a post-access stride, not an initial address displacement. +// CHECK-NOT: pto.addptr +// CHECK: scf.for +// CHECK: pto.vldus {{.*}}, %c64 {{.*}} -> !pto.vreg<64xf32>, !pto.align, !pto.ptr +// CHECK: pto.vstus {{.*}}, %c64_i32, {{.*}} -> !pto.align, !pto.ptr +// CHECK: scf.yield + +// CHECK-LABEL: func.func @sequential_vldus +// CHECK: %{{.*}}, %{{.*}}, [[BASE1:%[a-zA-Z0-9_]+]] = pto.vldus {{.*}}, %c32 +// CHECK: %{{.*}}, %{{.*}}, [[BASE2:%[a-zA-Z0-9_]+]] = pto.vldus [[BASE1]], {{.*}}, %c32 +// CHECK: %{{.*}}, %{{.*}} = pto.vldus [[BASE2]], {{.*}} + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @loop_vldus_vstus(%src: !pto.ptr, + %dst: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c64 = arith.constant 64 : index + %c192 = arith.constant 192 : index + %c64_i32 = arith.constant 64 : i32 + pto.vecscope { + %load_align = pto.vldas %src : !pto.ptr -> !pto.align + %store_align = pto.init_align : !pto.align + %final_src, %final_dst, %final_load_state, %final_store_state = + scf.for %iv = %c0 to %c192 step %c64 + iter_args(%src_iter = %src, %dst_iter = %dst, + %load_state = %load_align, %store_state = %store_align) + -> (!pto.ptr, !pto.ptr, !pto.align, !pto.align) { + %value, %next_load_state = pto.vldus %src_iter, %load_state + : !pto.ptr, !pto.align -> !pto.vreg<64xf32>, !pto.align + %next_store_state = pto.vstus %store_state, %c64_i32, %value, %dst_iter + : !pto.align, i32, !pto.vreg<64xf32>, !pto.ptr -> !pto.align + %next_src = pto.addptr %src_iter, %c64 : !pto.ptr -> !pto.ptr + %next_dst = pto.addptr %dst_iter, %c64 : !pto.ptr -> !pto.ptr + scf.yield %next_src, %next_dst, %next_load_state, %next_store_state + : !pto.ptr, !pto.ptr, !pto.align, !pto.align + } + pto.vstar %final_store_state, %final_dst : !pto.align, !pto.ptr + } + return + } + + func.func @sequential_vldus(%src: !pto.ptr, + %dst: !pto.ptr, + %x: index, %y: index) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c16 = arith.constant 16 : index + %c32 = arith.constant 32 : index + %c64 = arith.constant 64 : index + %x16 = arith.addi %x, %c16 : index + %y16 = arith.addi %y, %c16 : index + %x32 = arith.addi %x, %c32 : index + %y32 = arith.addi %y, %c32 : index + %base0a = pto.addptr %src, %x : !pto.ptr -> !pto.ptr + %base0 = pto.addptr %base0a, %y : !pto.ptr -> !pto.ptr + %base1a = pto.addptr %src, %x16 : !pto.ptr -> !pto.ptr + %base1 = pto.addptr %base1a, %y16 : !pto.ptr -> !pto.ptr + %base2a = pto.addptr %src, %x32 : !pto.ptr -> !pto.ptr + %base2 = pto.addptr %base2a, %y32 : !pto.ptr -> !pto.ptr + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %align0 = pto.vldas %base0 : !pto.ptr -> !pto.align + %v0, %align1 = pto.vldus %base0, %align0 + : !pto.ptr, !pto.align -> !pto.vreg<64xf32>, !pto.align + %v1, %align2 = pto.vldus %base1, %align1 + : !pto.ptr, !pto.align -> !pto.vreg<64xf32>, !pto.align + %v2, %align3 = pto.vldus %base2, %align2 + : !pto.ptr, !pto.align -> !pto.vreg<64xf32>, !pto.align + pto.vsts %v0, %dst[%c0], %mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + pto.vsts %v1, %dst[%c32], %mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + pto.vsts %v2, %dst[%c64], %mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } +} diff --git a/test/lit/vpto/spr_vstas_post_update_result_vpto_llvm.pto b/test/lit/vpto/spr_vstas_post_update_result_vpto_llvm.pto new file mode 100644 index 0000000000..46a999bcf9 --- /dev/null +++ b/test/lit/vpto/spr_vstas_post_update_result_vpto_llvm.pto @@ -0,0 +1,37 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software; you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: ptoas --cann-output-version=9.0.0 --pto-arch=a5 --pto-backend=vpto --emit-vpto-llvm-ir %s -o - 2>&1 | FileCheck %s +// RUN: ptoas --cann-output-version=9.0.0-beta.1 --pto-arch=a5 --pto-backend=vpto --emit-vpto-llvm-ir %s -o - 2>&1 | FileCheck %s + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @spr_vstas_post_update( + %spr_dst: !pto.ptr, %vstas_dst: !pto.ptr) attributes {pto.kernel} { + %c4 = arith.constant 4 : i32 + pto.vecscope { + %align = pto.init_align : !pto.align + %next_spr_dst = pto.sprsti "AR", %spr_dst[%c4] + : !pto.ptr, i32 -> !pto.ptr + %final_spr_dst = pto.sprsts "AR", %next_spr_dst[%c4] + : !pto.ptr, i32 -> !pto.ptr + %next_vstas_dst = pto.vstas %align, %vstas_dst, %c4 + : !pto.align, !pto.ptr, i32 -> !pto.ptr + } + return + } +} + +// CHECK-DAG: declare ptr addrspace(6) @llvm.hivm.sprsti.post +// CHECK-DAG: declare ptr addrspace(6) @llvm.hivm.sprsts.post +// CHECK-DAG: declare ptr addrspace(6) @llvm.hivm.vstas.post +// CHECK: call ptr addrspace(6) @llvm.hivm.sprsti.post(i16 74, +// CHECK-SAME: i32 4, i32 1) +// CHECK: call ptr addrspace(6) @llvm.hivm.sprsts.post(i16 74, +// CHECK-SAME: i32 4, i32 1) +// CHECK: call ptr addrspace(6) @llvm.hivm.vstas.post +// CHECK-SAME: i32 16, i32 1) diff --git a/test/lit/vpto/spr_vstas_post_update_verify_invalid.pto b/test/lit/vpto/spr_vstas_post_update_verify_invalid.pto new file mode 100644 index 0000000000..0299e3dba3 --- /dev/null +++ b/test/lit/vpto/spr_vstas_post_update_verify_invalid.pto @@ -0,0 +1,54 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software; you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: not pto-test-opt %s -split-input-file 2>&1 | FileCheck %s + +module { + func.func @sprsti_requires_matching_updated_base(%dst: !pto.ptr) { + %c0 = arith.constant 0 : i32 + pto.vecscope { + %next = pto.sprsti "AR", %dst[%c0] + : !pto.ptr, i32 -> !pto.ptr + } + return + } +} + +// CHECK: 'pto.sprsti' op requires updated base result to match base type + +// ----- + +module { + func.func @sprsts_requires_matching_updated_base(%dst: !pto.ptr) { + %c0 = arith.constant 0 : i32 + pto.vecscope { + %next = pto.sprsts "AR", %dst[%c0] + : !pto.ptr, i32 -> !pto.ptr + } + return + } +} + +// CHECK: 'pto.sprsts' op requires updated base result to match base type + +// ----- + +module { + func.func @vstas_requires_matching_updated_base( + %dst: !pto.ptr) { + %c0 = arith.constant 0 : i32 + pto.vecscope { + %align = pto.init_align : !pto.align + %next = pto.vstas %align, %dst, %c0 + : !pto.align, !pto.ptr, i32 -> !pto.ptr + } + return + } +} + +// CHECK: 'pto.vstas' op requires updated base result to match base type diff --git a/test/lit/vpto/vbitcast_vpto_llvm.pto b/test/lit/vpto/vbitcast_vpto_llvm.pto index cb921563a8..59b0b2015c 100644 --- a/test/lit/vpto/vbitcast_vpto_llvm.pto +++ b/test/lit/vpto/vbitcast_vpto_llvm.pto @@ -34,6 +34,6 @@ module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind&1 | FileCheck %s +// RUN: ptoas --cann-output-version=9.0.0-beta.1 --pto-arch=a5 --pto-backend=vpto --emit-vpto-llvm-ir %s -o - 2>&1 | FileCheck %s + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @load_post_update(%src: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c1_i16 = arith.constant 1 : i16 + %c2_i16 = arith.constant 2 : i16 + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %low, %high, %next_x2 = pto.vldsx2 %src[%c0], "DINTLV_B32" + : !pto.ptr, index + -> !pto.vreg<64xf32>, !pto.vreg<64xf32>, !pto.ptr + %value, %next_block = pto.vsldb %next_x2, %c2_i16, %c1_i16, %mask + : !pto.ptr, i16, i16, !pto.mask + -> !pto.vreg<64xf32>, !pto.ptr + pto.vsts %low, %next_block[%c0], %mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + pto.vsts %high, %next_block[%c0], %mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + pto.vsts %value, %next_block[%c0], %mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } +} + +// CHECK-DAG: declare { <64 x float>, <64 x float>, ptr addrspace(6) } @llvm.hivm.vldsx2.post.v64f32 +// CHECK-DAG: declare { <64 x float>, ptr addrspace(6) } @llvm.hivm.vsldb.post.v64f32 +// CHECK: call { <64 x float>, <64 x float>, ptr addrspace(6) } @llvm.hivm.vldsx2.post.v64f32 +// CHECK-SAME: i32 1) +// CHECK: call { <64 x float>, ptr addrspace(6) } @llvm.hivm.vsldb.post.v64f32 +// CHECK-SAME: i32 1, diff --git a/test/lit/vpto/vldsx2_vsldb_post_update_verify_invalid.pto b/test/lit/vpto/vldsx2_vsldb_post_update_verify_invalid.pto new file mode 100644 index 0000000000..9370acaafb --- /dev/null +++ b/test/lit/vpto/vldsx2_vsldb_post_update_verify_invalid.pto @@ -0,0 +1,40 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software; you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: not pto-test-opt %s -split-input-file 2>&1 | FileCheck %s + +module { + func.func @vldsx2_requires_matching_updated_base(%src: !pto.ptr) { + %c0 = arith.constant 0 : index + pto.vecscope { + %low, %high, %next = pto.vldsx2 %src[%c0], "DINTLV_B32" + : !pto.ptr, index + -> !pto.vreg<64xf32>, !pto.vreg<64xf32>, !pto.ptr + } + return + } +} + +// CHECK: 'pto.vldsx2' op requires updated base result to match base type + +// ----- + +module { + func.func @vsldb_requires_matching_updated_base(%src: !pto.ptr) { + %c1_i16 = arith.constant 1 : i16 + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %value, %next = pto.vsldb %src, %c1_i16, %c1_i16, %mask + : !pto.ptr, i16, i16, !pto.mask + -> !pto.vreg<64xf32>, !pto.ptr + } + return + } +} + +// CHECK: 'pto.vsldb' op requires updated base result to match base type diff --git a/test/lit/vpto/vldus_vstus_post_update_result_vpto_llvm.pto b/test/lit/vpto/vldus_vstus_post_update_result_vpto_llvm.pto new file mode 100644 index 0000000000..534bd1c028 --- /dev/null +++ b/test/lit/vpto/vldus_vstus_post_update_result_vpto_llvm.pto @@ -0,0 +1,35 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: ptoas --cann-output-version=9.0.0 --pto-arch=a5 --pto-backend=vpto --emit-vpto-llvm-ir %s -o - 2>&1 | FileCheck %s +// RUN: ptoas --cann-output-version=9.0.0-beta.1 --pto-arch=a5 --pto-backend=vpto --emit-vpto-llvm-ir %s -o - 2>&1 | FileCheck %s + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @unaligned_post_update(%src: !pto.ptr, + %dst: !pto.ptr) attributes {pto.kernel} { + %c64 = arith.constant 64 : index + %c64_i32 = arith.constant 64 : i32 + pto.vecscope { + %load_align = pto.vldas %src : !pto.ptr -> !pto.align + %value, %load_align_out, %src_out = pto.vldus %src, %load_align, %c64 + : !pto.ptr, !pto.align, index + -> !pto.vreg<64xf32>, !pto.align, !pto.ptr + %store_align = pto.init_align : !pto.align + %store_align_out, %dst_out = pto.vstus %store_align, %c64_i32, %value, %dst + : !pto.align, i32, !pto.vreg<64xf32>, !pto.ptr + -> !pto.align, !pto.ptr + pto.vstar %store_align_out, %dst_out : !pto.align, !pto.ptr + } + return + } +} + +// CHECK-LABEL: define void @unaligned_post_update_mix_aiv +// CHECK: call <32 x i8> @llvm.hivm.vldas +// CHECK: call { <64 x float>, <32 x i8>, ptr addrspace(6) } @llvm.hivm.vldus.post.v64f32(ptr addrspace(6) {{.*}}, <32 x i8> {{.*}}, i32 256) +// CHECK: call { <32 x i8>, ptr addrspace(6) } @llvm.hivm.vstus.post.v64f32(<64 x float> {{.*}}, ptr addrspace(6) {{.*}}, i32 256, <32 x i8> {{.*}}) diff --git a/test/lit/vpto/vldus_vstus_post_update_verify_invalid.pto b/test/lit/vpto/vldus_vstus_post_update_verify_invalid.pto new file mode 100644 index 0000000000..2a2fa38a53 --- /dev/null +++ b/test/lit/vpto/vldus_vstus_post_update_verify_invalid.pto @@ -0,0 +1,24 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: not ptoas --pto-arch=a5 --pto-backend=vpto %s 2>&1 | FileCheck %s + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @vldus_post_update_requires_matching_source_type(%src: !pto.ptr) attributes {pto.kernel} { + %c64 = arith.constant 64 : index + pto.vecscope { + %align = pto.vldas %src : !pto.ptr -> !pto.align + %value, %align_out, %next = pto.vldus %src, %align, %c64 + : !pto.ptr, !pto.align, index + -> !pto.vreg<64xf32>, !pto.align, !pto.ptr + } + return + } +} + +// CHECK: requires updated base result to match source type diff --git a/test/lit/vpto/vreg_low_precision_memory_vpto_llvm.pto b/test/lit/vpto/vreg_low_precision_memory_vpto_llvm.pto index 2421fdfd43..37ba0cbb21 100644 --- a/test/lit/vpto/vreg_low_precision_memory_vpto_llvm.pto +++ b/test/lit/vpto/vreg_low_precision_memory_vpto_llvm.pto @@ -6,7 +6,8 @@ // INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. // See LICENSE in the root of the software repository for the full text of the License. -// RUN: ptoas --cann-output-version=9.0.0 --pto-arch=a5 --pto-backend=vpto --emit-vpto-llvm-ir %s -o - 2>&1 | FileCheck %s +// RUN: ptoas --cann-output-version=9.0.0 --pto-arch=a5 --pto-backend=vpto --emit-vpto-llvm-ir %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CANN +// RUN: ptoas --cann-output-version=9.0.0-beta.1 --pto-arch=a5 --pto-backend=vpto --emit-vpto-llvm-ir %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,BETA module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { func.func @lowp_vlds_vsts(%f8e4_src: !pto.ptr, @@ -30,7 +31,8 @@ module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind, %f4e2_dst: !pto.ptr, - %value: !pto.vreg<256x!pto.f4E2M1x2>) attributes {pto.kernel} { + %value: !pto.vreg<256x!pto.f4E2M1x2>, + %hif8_src: !pto.ptr) attributes {pto.kernel} { %c0 = arith.constant 0 : index %c1_i16 = arith.constant 1 : i16 pto.vecscope { @@ -40,6 +42,7 @@ module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind, i16, i16, !pto.mask -> !pto.vreg<256x!pto.f4E1M2x2> pto.vsstb %strided, %f4e2_dst, %c1_i16, %c1_i16, %mask : !pto.vreg<256x!pto.f4E1M2x2>, !pto.ptr, i16, i16, !pto.mask pto.vsstb %value, %f4e2_dst, %c1_i16, %c1_i16, %mask : !pto.vreg<256x!pto.f4E2M1x2>, !pto.ptr, i16, i16, !pto.mask + %hif8_strided = pto.vsldb %hif8_src, %c1_i16, %c1_i16, %mask : !pto.ptr, i16, i16, !pto.mask -> !pto.vreg<256x!pto.hif8> } return } @@ -74,10 +77,12 @@ module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind @llvm.hivm.vldsx1.v256f8e4m3 // CHECK: call <256 x i8> @llvm.hivm.vldsx1.v256f8e5m2 -// CHECK: call <256 x i8> @llvm.hivm.vldsx1.v256s8 +// CANN: call <256 x i8> @llvm.hivm.vldsx1.v256s8 +// BETA: call <256 x i8> @llvm.hivm.vldsx1.v256hif8 // CHECK: call void @llvm.hivm.vstsx1.v256f8e4m3 // CHECK: call void @llvm.hivm.vstsx1.v256f8e5m2 -// CHECK: call void @llvm.hivm.vstsx1.v256s8 +// CANN: call void @llvm.hivm.vstsx1.v256s8 +// BETA: call void @llvm.hivm.vstsx1.v256hif8 // CHECK-LABEL: define void @lowp_x2_and_block_mix_aiv // CHECK: call { <256 x i8>, <256 x i8> } @llvm.hivm.vldsx2.v256f4e1m2x2 @@ -88,7 +93,8 @@ module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind @llvm.hivm.vldas -// CHECK: call { <256 x i8>, <32 x i8>, ptr addrspace(6) } @llvm.hivm.vldus.v256s8 +// CANN: call { <256 x i8>, <32 x i8>, ptr addrspace(6) } @llvm.hivm.vldus.v256s8 +// BETA: call { <256 x i8>, <32 x i8>, ptr addrspace(6) } @llvm.hivm.vldus.v256hif8 // CHECK: call <32 x i8> @llvm.hivm.vstus // CHECK: call <32 x i8> @llvm.hivm.vstur // CHECK: call void @llvm.hivm.vstar diff --git a/test/vpto/cases/micro-op/predicate-load-store/predicate-post-update/compare.py b/test/vpto/cases/micro-op/predicate-load-store/predicate-post-update/compare.py new file mode 100644 index 0000000000..3d1d17d02b --- /dev/null +++ b/test/vpto/cases/micro-op/predicate-load-store/predicate-post-update/compare.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import numpy as np + + +def main() -> None: + golden = np.fromfile("golden_output.bin", dtype=np.uint8) + output = np.fromfile("output.bin", dtype=np.uint8) + if not np.array_equal(golden, output): + mismatch = np.flatnonzero(golden != output) + idx = int(mismatch[0]) if mismatch.size else 0 + print( + f"[ERROR] mismatch: idx={idx}, " + f"golden={int(golden[idx]) if golden.size else 'n/a'}, " + f"output={int(output[idx]) if output.size else 'n/a'}" + ) + raise SystemExit(2) + print("[INFO] compare passed") + + +if __name__ == "__main__": + main() diff --git a/test/vpto/cases/micro-op/predicate-load-store/predicate-post-update/golden.py b/test/vpto/cases/micro-op/predicate-load-store/predicate-post-update/golden.py new file mode 100644 index 0000000000..7a01c76439 --- /dev/null +++ b/test/vpto/cases/micro-op/predicate-load-store/predicate-post-update/golden.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import numpy as np + + +def main() -> None: + indices = np.arange(128, dtype=np.uint16) + data = ((indices * 37 + 11) & 0xFF).astype(np.uint8) + data.tofile("input.bin") + np.zeros((128,), dtype=np.uint8).tofile("output.bin") + data.tofile("golden_output.bin") + + +if __name__ == "__main__": + main() diff --git a/test/vpto/cases/micro-op/predicate-load-store/predicate-post-update/kernel.pto b/test/vpto/cases/micro-op/predicate-load-store/predicate-post-update/kernel.pto new file mode 100644 index 0000000000..3111724e7e --- /dev/null +++ b/test/vpto/cases/micro-op/predicate-load-store/predicate-post-update/kernel.pto @@ -0,0 +1,71 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// case: micro-op/predicate-load-store/predicate-post-update +// family: predicate-load-store +// target_ops: pto.plds, pto.pldi, pto.psts, pto.psti +// scenarios: post-update-result, immediate-block-stride, scalar-byte-stride + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @predicate_post_update_kernel( + %input: !pto.ptr, %output: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c32 = arith.constant 32 : index + %c64 = arith.constant 64 : index + %c0_i64 = arith.constant 0 : i64 + %c4_i64 = arith.constant 4 : i64 + %c32_i64 = arith.constant 32 : i64 + %c4096_i64 = arith.constant 4096 : i64 + + %ub_in = pto.castptr %c0_i64 : i64 -> !pto.ptr + %ub_out = pto.castptr %c4096_i64 : i64 -> !pto.ptr + %ub_in_scalar = pto.addptr %ub_in, %c64 + : !pto.ptr -> !pto.ptr + %ub_out_scalar = pto.addptr %ub_out, %c64 + : !pto.ptr -> !pto.ptr + + pto.mte_gm_ub %input, %ub_in, %c0_i64, %c32_i64 + nburst(%c4_i64, %c32_i64, %c32_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64, i64 + pto.mte_gm_ub %output, %ub_out, %c0_i64, %c32_i64 + nburst(%c4_i64, %c32_i64, %c32_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64, i64 + + pto.set_flag["PIPE_MTE2", "PIPE_V", "EVENT_ID0"] + pto.wait_flag["PIPE_MTE2", "PIPE_V", "EVENT_ID0"] + + pto.vecscope { + %mask_i0, %src_i1 = pto.pldi %ub_in[%c1], "NORM" + : !pto.ptr, index -> !pto.mask, !pto.ptr + %mask_i1, %src_i2 = pto.pldi %src_i1[%c1], "NORM" + : !pto.ptr, index -> !pto.mask, !pto.ptr + %dst_i1 = pto.psti %mask_i0, %ub_out[%c1], "NORM" + : !pto.mask, !pto.ptr, index -> !pto.ptr + %dst_i2 = pto.psti %mask_i1, %dst_i1[%c1], "NORM" + : !pto.mask, !pto.ptr, index -> !pto.ptr + + %mask_s0, %src_s1 = pto.plds %ub_in_scalar[%c32], "NORM" + : !pto.ptr, index -> !pto.mask, !pto.ptr + %mask_s1, %src_s2 = pto.plds %src_s1[%c32], "NORM" + : !pto.ptr, index -> !pto.mask, !pto.ptr + %dst_s1 = pto.psts %mask_s0, %ub_out_scalar[%c32], "NORM" + : !pto.mask, !pto.ptr, index -> !pto.ptr + %dst_s2 = pto.psts %mask_s1, %dst_s1[%c32], "NORM" + : !pto.mask, !pto.ptr, index -> !pto.ptr + } + + pto.set_flag["PIPE_V", "PIPE_MTE3", "EVENT_ID0"] + pto.wait_flag["PIPE_V", "PIPE_MTE3", "EVENT_ID0"] + pto.mte_ub_gm %ub_out, %output, %c32_i64 + nburst(%c4_i64, %c32_i64, %c32_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64 + pto.barrier #pto.pipe + return + } +} diff --git a/test/vpto/cases/micro-op/predicate-load-store/predicate-post-update/launch.cpp b/test/vpto/cases/micro-op/predicate-load-store/predicate-post-update/launch.cpp new file mode 100644 index 0000000000..9e2a045ee5 --- /dev/null +++ b/test/vpto/cases/micro-op/predicate-load-store/predicate-post-update/launch.cpp @@ -0,0 +1,27 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#ifndef __VEC_SCOPE__ +#define __VEC_SCOPE__ +#endif + +#include + +#ifndef __CPU_SIM +#include "acl/acl.h" +#endif + +extern "C" __global__[aicore] void +predicate_post_update_kernel(__gm__ unsigned char *input, + __gm__ unsigned char *output); + +void LaunchPredicatePostUpdate(unsigned char *input, unsigned char *output, + void *stream) { + predicate_post_update_kernel<<<1, nullptr, stream>>>( + (__gm__ unsigned char *)input, (__gm__ unsigned char *)output); +} diff --git a/test/vpto/cases/micro-op/predicate-load-store/predicate-post-update/main.cpp b/test/vpto/cases/micro-op/predicate-load-store/predicate-post-update/main.cpp new file mode 100644 index 0000000000..092c7d1755 --- /dev/null +++ b/test/vpto/cases/micro-op/predicate-load-store/predicate-post-update/main.cpp @@ -0,0 +1,90 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#include "acl/acl.h" +#include "test_common.h" + +#include +#include + +using namespace PtoTestCommon; + +void LaunchPredicatePostUpdate(unsigned char *input, unsigned char *output, + void *stream); + +namespace { +constexpr size_t kBufferBytes = 128; +} // namespace + +#define ACL_CHECK(expr) \ + do { \ + const aclError ret = (expr); \ + if (ret != ACL_SUCCESS) { \ + std::fprintf(stderr, "[ERROR] %s failed: %d (%s:%d)\n", #expr, \ + static_cast(ret), __FILE__, __LINE__); \ + rc = 1; \ + goto cleanup; \ + } \ + } while (0) + +int main() { + unsigned char *inputHost = nullptr; + unsigned char *outputHost = nullptr; + unsigned char *inputDevice = nullptr; + unsigned char *outputDevice = nullptr; + aclrtStream stream = nullptr; + int rc = 0; + bool aclInited = false; + bool deviceSet = false; + int deviceId = 0; + size_t inputSize = kBufferBytes; + size_t outputSize = kBufferBytes; + + ACL_CHECK(aclInit(nullptr)); + aclInited = true; + if (const char *envDevice = std::getenv("ACL_DEVICE_ID")) + deviceId = std::atoi(envDevice); + ACL_CHECK(aclrtSetDevice(deviceId)); + deviceSet = true; + ACL_CHECK(aclrtCreateStream(&stream)); + + ACL_CHECK( + aclrtMallocHost(reinterpret_cast(&inputHost), kBufferBytes)); + ACL_CHECK( + aclrtMallocHost(reinterpret_cast(&outputHost), kBufferBytes)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&inputDevice), kBufferBytes, + ACL_MEM_MALLOC_HUGE_FIRST)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&outputDevice), kBufferBytes, + ACL_MEM_MALLOC_HUGE_FIRST)); + + ReadFile("./input.bin", inputSize, inputHost, kBufferBytes); + ReadFile("./output.bin", outputSize, outputHost, kBufferBytes); + ACL_CHECK(aclrtMemcpy(inputDevice, kBufferBytes, inputHost, kBufferBytes, + ACL_MEMCPY_HOST_TO_DEVICE)); + ACL_CHECK(aclrtMemcpy(outputDevice, kBufferBytes, outputHost, kBufferBytes, + ACL_MEMCPY_HOST_TO_DEVICE)); + + LaunchPredicatePostUpdate(inputDevice, outputDevice, stream); + ACL_CHECK(aclrtSynchronizeStream(stream)); + ACL_CHECK(aclrtMemcpy(outputHost, kBufferBytes, outputDevice, kBufferBytes, + ACL_MEMCPY_DEVICE_TO_HOST)); + WriteFile("./output.bin", outputHost, kBufferBytes); + +cleanup: + aclrtFree(inputDevice); + aclrtFree(outputDevice); + aclrtFreeHost(inputHost); + aclrtFreeHost(outputHost); + if (stream != nullptr) + aclrtDestroyStream(stream); + if (deviceSet) + aclrtResetDevice(deviceId); + if (aclInited) + aclFinalize(); + return rc; +} diff --git a/test/vpto/cases/micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment/compare.py b/test/vpto/cases/micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment/compare.py new file mode 100644 index 0000000000..2bace0619e --- /dev/null +++ b/test/vpto/cases/micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment/compare.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import numpy as np + + +def main() -> None: + golden = np.fromfile("golden_output.bin", dtype=np.uint8) + output = np.fromfile("output.bin", dtype=np.uint8) + if golden.shape != output.shape or not np.array_equal(golden, output): + mismatch = np.flatnonzero(golden != output) + idx = int(mismatch[0]) if mismatch.size else 0 + print( + f"[ERROR] predicate soft-postupdate mismatch: idx={idx}, " + f"golden={int(golden[idx]) if golden.size else 'n/a'}, " + f"output={int(output[idx]) if output.size else 'n/a'}" + ) + raise SystemExit(2) + print("[INFO] predicate soft-postupdate DIST compare passed") + + +if __name__ == "__main__": + main() diff --git a/test/vpto/cases/micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment/golden.py b/test/vpto/cases/micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment/golden.py new file mode 100644 index 0000000000..2ad1173a6c --- /dev/null +++ b/test/vpto/cases/micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment/golden.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import numpy as np + + +BUFFER_BYTES = 2048 +NORM_BASE = 0 +US_BASE = 512 +DS_BASE = 1024 + + +def downsample_predicate(payload: np.ndarray) -> np.ndarray: + bits = np.unpackbits(payload, bitorder="little") + return np.packbits(bits[::2], bitorder="little") + + +def main() -> None: + indices = np.arange(BUFFER_BYTES, dtype=np.uint32) + data = ((indices * 73 + 19) & 0xFF).astype(np.uint8) + golden = np.zeros((BUFFER_BYTES,), dtype=np.uint8) + + # Two normal loop iterations must advance by 32 bytes in NORM mode. + golden[NORM_BASE : NORM_BASE + 64] = data[NORM_BASE : NORM_BASE + 64] + + # US expands a 16-byte packed image and PK packs it back to 16 bytes. + golden[US_BASE : US_BASE + 32] = data[US_BASE : US_BASE + 32] + + # DS consumes 64 bytes and keeps every other bit. The two source windows + # begin 32 bytes apart, exactly matching the A5 DS alignment unit. + for iteration in range(2): + src = DS_BASE + iteration * 32 + dst = DS_BASE + iteration * 32 + golden[dst : dst + 32] = downsample_predicate(data[src : src + 64]) + + data.tofile("input.bin") + np.zeros((BUFFER_BYTES,), dtype=np.uint8).tofile("output.bin") + golden.tofile("golden_output.bin") + + +if __name__ == "__main__": + main() diff --git a/test/vpto/cases/micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment/kernel.pto b/test/vpto/cases/micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment/kernel.pto new file mode 100644 index 0000000000..6c35e2e4bc --- /dev/null +++ b/test/vpto/cases/micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment/kernel.pto @@ -0,0 +1,112 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// case: micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment +// family: predicate-load-store +// target_ops: pto.pldi, pto.psti +// target_pass: vpto-soft-postupdate +// scenarios: normal-loop-recurrence, dist-dependent-alignment, strict-output + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @predicate_soft_postupdate_dist_alignment_kernel( + %input: !pto.ptr, + %output: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c16 = arith.constant 16 : index + %c32 = arith.constant 32 : index + %c512 = arith.constant 512 : index + %c1024 = arith.constant 1024 : index + %c0_i64 = arith.constant 0 : i64 + %c16_i64 = arith.constant 16 : i64 + %c128_i64 = arith.constant 128 : i64 + %c4096_i64 = arith.constant 4096 : i64 + + %ub_input = pto.castptr %c0_i64 : i64 -> !pto.ptr + %ub_output = pto.castptr %c4096_i64 : i64 -> !pto.ptr + %ub_input_us = pto.addptr %ub_input, %c512 + : !pto.ptr -> !pto.ptr + %ub_input_ds = pto.addptr %ub_input, %c1024 + : !pto.ptr -> !pto.ptr + %ub_output_us = pto.addptr %ub_output, %c512 + : !pto.ptr -> !pto.ptr + %ub_output_ds = pto.addptr %ub_output, %c1024 + : !pto.ptr -> !pto.ptr + + pto.mte_gm_ub %input, %ub_input, %c0_i64, %c128_i64 + nburst(%c16_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64, i64 + pto.mte_gm_ub %output, %ub_output, %c0_i64, %c128_i64 + nburst(%c16_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64, i64 + + pto.set_flag["PIPE_MTE2", "PIPE_V", "EVENT_ID0"] + pto.wait_flag["PIPE_MTE2", "PIPE_V", "EVENT_ID0"] + + pto.vecscope { + // The source contains only normal forms. Each software recurrence is one + // DIST-selected alignment unit and must become a post stride of one. + scf.for %iv = %c0 to %c2 step %c1 + iter_args(%src = %ub_input, %dst = %ub_output) + -> (!pto.ptr, !pto.ptr) { + %mask = pto.pldi %src[%c0], "NORM" + : !pto.ptr, index -> !pto.mask + pto.mem_bar "VST_VLD" + pto.psti %mask, %dst[%c0], "NORM" + : !pto.mask, !pto.ptr, index + %next_src = pto.addptr %src, %c32 + : !pto.ptr -> !pto.ptr + %next_dst = pto.addptr %dst, %c32 + : !pto.ptr -> !pto.ptr + scf.yield %next_src, %next_dst + : !pto.ptr, !pto.ptr + } + + scf.for %iv = %c0 to %c2 step %c1 + iter_args(%src = %ub_input_us, %dst = %ub_output_us) + -> (!pto.ptr, !pto.ptr) { + %mask = pto.pldi %src[%c0], "US" + : !pto.ptr, index -> !pto.mask + pto.mem_bar "VST_VLD" + pto.psti %mask, %dst[%c0], "PK" + : !pto.mask, !pto.ptr, index + %next_src = pto.addptr %src, %c16 + : !pto.ptr -> !pto.ptr + %next_dst = pto.addptr %dst, %c16 + : !pto.ptr -> !pto.ptr + scf.yield %next_src, %next_dst + : !pto.ptr, !pto.ptr + } + + scf.for %iv = %c0 to %c2 step %c1 + iter_args(%src = %ub_input_ds, %dst = %ub_output_ds) + -> (!pto.ptr, !pto.ptr) { + %mask = pto.pldi %src[%c0], "DS" + : !pto.ptr, index -> !pto.mask + pto.mem_bar "VST_VLD" + pto.psti %mask, %dst[%c0], "NORM" + : !pto.mask, !pto.ptr, index + %next_src = pto.addptr %src, %c32 + : !pto.ptr -> !pto.ptr + %next_dst = pto.addptr %dst, %c32 + : !pto.ptr -> !pto.ptr + scf.yield %next_src, %next_dst + : !pto.ptr, !pto.ptr + } + } + + pto.set_flag["PIPE_V", "PIPE_MTE3", "EVENT_ID0"] + pto.wait_flag["PIPE_V", "PIPE_MTE3", "EVENT_ID0"] + pto.mte_ub_gm %ub_output, %output, %c128_i64 + nburst(%c16_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64 + pto.barrier #pto.pipe + return + } +} diff --git a/test/vpto/cases/micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment/launch.cpp b/test/vpto/cases/micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment/launch.cpp new file mode 100644 index 0000000000..2bcd46153d --- /dev/null +++ b/test/vpto/cases/micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment/launch.cpp @@ -0,0 +1,28 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#ifndef __VEC_SCOPE__ +#define __VEC_SCOPE__ +#endif + +#include + +#ifndef __CPU_SIM +#include "acl/acl.h" +#endif + +extern "C" __global__[aicore] void +predicate_soft_postupdate_dist_alignment_kernel(__gm__ uint8_t *input, + __gm__ uint8_t *output); + +void LaunchPredicateSoftPostupdateDistAlignment(uint8_t *input, + uint8_t *output, + void *stream) { + predicate_soft_postupdate_dist_alignment_kernel<<<1, nullptr, stream>>>( + (__gm__ uint8_t *)input, (__gm__ uint8_t *)output); +} diff --git a/test/vpto/cases/micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment/main.cpp b/test/vpto/cases/micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment/main.cpp new file mode 100644 index 0000000000..bbd8e39ffb --- /dev/null +++ b/test/vpto/cases/micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment/main.cpp @@ -0,0 +1,92 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#include "acl/acl.h" +#include "test_common.h" + +#include +#include +#include + +using namespace PtoTestCommon; + +void LaunchPredicateSoftPostupdateDistAlignment(uint8_t *input, + uint8_t *output, + void *stream); + +namespace { +constexpr size_t kBufferBytes = 2048; +} // namespace + +#define ACL_CHECK(expr) \ + do { \ + const aclError ret = (expr); \ + if (ret != ACL_SUCCESS) { \ + std::fprintf(stderr, "[ERROR] %s failed: %d (%s:%d)\n", #expr, \ + static_cast(ret), __FILE__, __LINE__); \ + rc = 1; \ + goto cleanup; \ + } \ + } while (0) + +int main() { + uint8_t *inputHost = nullptr; + uint8_t *outputHost = nullptr; + uint8_t *inputDevice = nullptr; + uint8_t *outputDevice = nullptr; + aclrtStream stream = nullptr; + int rc = 0; + bool aclInited = false; + bool deviceSet = false; + int deviceId = 0; + size_t inputSize = kBufferBytes; + size_t outputSize = kBufferBytes; + + ACL_CHECK(aclInit(nullptr)); + aclInited = true; + if (const char *envDevice = std::getenv("ACL_DEVICE_ID")) + deviceId = std::atoi(envDevice); + ACL_CHECK(aclrtSetDevice(deviceId)); + deviceSet = true; + ACL_CHECK(aclrtCreateStream(&stream)); + + ACL_CHECK( + aclrtMallocHost(reinterpret_cast(&inputHost), kBufferBytes)); + ACL_CHECK( + aclrtMallocHost(reinterpret_cast(&outputHost), kBufferBytes)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&inputDevice), kBufferBytes, + ACL_MEM_MALLOC_HUGE_FIRST)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&outputDevice), kBufferBytes, + ACL_MEM_MALLOC_HUGE_FIRST)); + + ReadFile("./input.bin", inputSize, inputHost, kBufferBytes); + ReadFile("./output.bin", outputSize, outputHost, kBufferBytes); + ACL_CHECK(aclrtMemcpy(inputDevice, kBufferBytes, inputHost, kBufferBytes, + ACL_MEMCPY_HOST_TO_DEVICE)); + ACL_CHECK(aclrtMemcpy(outputDevice, kBufferBytes, outputHost, kBufferBytes, + ACL_MEMCPY_HOST_TO_DEVICE)); + + LaunchPredicateSoftPostupdateDistAlignment(inputDevice, outputDevice, stream); + ACL_CHECK(aclrtSynchronizeStream(stream)); + ACL_CHECK(aclrtMemcpy(outputHost, kBufferBytes, outputDevice, kBufferBytes, + ACL_MEMCPY_DEVICE_TO_HOST)); + WriteFile("./output.bin", outputHost, kBufferBytes); + +cleanup: + aclrtFree(inputDevice); + aclrtFree(outputDevice); + aclrtFreeHost(inputHost); + aclrtFreeHost(outputHost); + if (stream != nullptr) + aclrtDestroyStream(stream); + if (deviceSet) + aclrtResetDevice(deviceId); + if (aclInited) + aclFinalize(); + return rc; +} diff --git a/test/vpto/cases/micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment/ptoas.flags b/test/vpto/cases/micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment/ptoas.flags new file mode 100644 index 0000000000..28c4691e94 --- /dev/null +++ b/test/vpto/cases/micro-op/predicate-load-store/predicate-soft-postupdate-dist-alignment/ptoas.flags @@ -0,0 +1 @@ +--pto-arch a5 --pto-backend=vpto --enable-vpto-soft-postupdate diff --git a/test/vpto/cases/micro-op/vector-load-store/soft-post-update-address-units/compare.py b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-address-units/compare.py new file mode 100644 index 0000000000..1a00a94a56 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-address-units/compare.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import numpy as np + + +def check(name: str, dtype: np.dtype) -> None: + suffix = "" if name == "main" else f"_{name}" + golden = np.fromfile(f"golden_output{suffix}.bin", dtype=dtype) + output = np.fromfile(f"output{suffix}.bin", dtype=dtype) + if golden.shape != output.shape or not np.array_equal(golden, output): + mismatch = np.flatnonzero(golden != output) + idx = int(mismatch[0]) if mismatch.size else 0 + print( + f"[ERROR] {name} mismatch: shape={output.shape}, idx={idx}, " + f"golden={golden[idx] if golden.size else 'n/a'}, " + f"output={output[idx] if output.size else 'n/a'}" + ) + raise SystemExit(2) + + +def main() -> None: + check("main", np.float32) + check("spr", np.uint32) + print("[INFO] compare passed") + + +if __name__ == "__main__": + main() diff --git a/test/vpto/cases/micro-op/vector-load-store/soft-post-update-address-units/golden.py b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-address-units/golden.py new file mode 100644 index 0000000000..8e7699afbb --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-address-units/golden.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import argparse +from pathlib import Path + +import numpy as np + + +ELEMENTS = 1024 +SEED = 29 + + +def generate(output_dir: Path, seed: int) -> None: + rng = np.random.default_rng(seed) + data = rng.uniform(-16.0, 16.0, size=(ELEMENTS,)).astype(np.float32) + output = np.zeros((ELEMENTS,), dtype=np.float32) + spr_input = (np.arange(ELEMENTS, dtype=np.uint32) * 17) + 3 + spr_output = np.zeros((ELEMENTS,), dtype=np.uint32) + + golden_spr = spr_input.copy() + # The initial base is element 4 and the signed immediate is -1 word. The + # post-update recurrence therefore stores cleared AR values at [3, 7). + golden_spr[3:7] = 0 + + output_dir.mkdir(parents=True, exist_ok=True) + data.tofile(output_dir / "input.bin") + output.tofile(output_dir / "output.bin") + data.tofile(output_dir / "golden_output.bin") + spr_input.tofile(output_dir / "input_spr.bin") + spr_output.tofile(output_dir / "output_spr.bin") + golden_spr.tofile(output_dir / "golden_output_spr.bin") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Generate data for soft post-update address-unit validation." + ) + parser.add_argument("--output-dir", type=Path, default=Path(".")) + parser.add_argument("--seed", type=int, default=SEED) + args = parser.parse_args() + generate(args.output_dir, args.seed) + + +if __name__ == "__main__": + main() diff --git a/test/vpto/cases/micro-op/vector-load-store/soft-post-update-address-units/kernel.pto b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-address-units/kernel.pto new file mode 100644 index 0000000000..e40054b794 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-address-units/kernel.pto @@ -0,0 +1,89 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// case: micro-op/vector-load-store/soft-post-update-address-units +// family: vector-load-store +// target_pass: vpto-soft-postupdate +// scenarios: mixed-element-byte-units, signed-word-initial-offset + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @soft_post_update_address_units_kernel( + %input: !pto.ptr, + %output: !pto.ptr, + %spr_input: !pto.ptr, + %spr_output: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c4 = arith.constant 4 : index + %c64 = arith.constant 64 : index + %c1024 = arith.constant 1024 : index + %c64_i32 = arith.constant 64 : i32 + %cm1_i32 = arith.constant -1 : i32 + %c0_i64 = arith.constant 0 : i64 + %c32_i64 = arith.constant 32 : i64 + %c128_i64 = arith.constant 128 : i64 + %c4096_i64 = arith.constant 4096 : i64 + %c12288_i64 = arith.constant 12288 : i64 + + %ub_in = pto.castptr %c0_i64 : i64 -> !pto.ptr + %ub_out = pto.castptr %c4096_i64 : i64 -> !pto.ptr + %ub_spr_raw = pto.castptr %c12288_i64 : i64 -> !pto.ptr + %ub_spr = pto.addptr %ub_spr_raw, %c4 + : !pto.ptr -> !pto.ptr + + pto.mte_gm_ub %input, %ub_in, %c0_i64, %c128_i64 + nburst(%c32_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64, i64 + pto.mte_gm_ub %spr_input, %ub_spr_raw, %c0_i64, %c128_i64 + nburst(%c32_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64, i64 + + pto.set_flag["PIPE_MTE2", "PIPE_V", "EVENT_ID0"] + pto.wait_flag["PIPE_MTE2", "PIPE_V", "EVENT_ID0"] + + pto.vecscope { + %vector_mask, %remaining = pto.plt_b32 %c64_i32 + : i32 -> !pto.mask, i32 + %predicate_mask = pto.pset_b8 "PAT_ALL" : !pto.mask + + // vlds advances in f32 elements while psts advances in bytes. They use + // the same base and IV deliberately and must not share an iter_arg. + scf.for %iv = %c0 to %c1024 step %c64 { + %value = pto.vlds %ub_in[%iv] + : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %value, %ub_out[%iv], %vector_mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + pto.psts %predicate_mask, %ub_in[%iv], "NORM" + : !pto.mask, !pto.ptr, index + } + + // The signed immediate is a 4-byte word count. Starting one element + // past the raw base and storing at -1 exercises signed word recurrence. + scf.for %iv = %c0 to %c4 step %c1 + iter_args(%ptr = %ub_spr) -> (!pto.ptr) { + pto.sprclr "AR" + pto.sprsti "AR", %ptr[%cm1_i32] + : !pto.ptr, i32 + %next_ptr = pto.addptr %ptr, %c1 + : !pto.ptr -> !pto.ptr + scf.yield %next_ptr : !pto.ptr + } + } + + pto.set_flag["PIPE_V", "PIPE_MTE3", "EVENT_ID0"] + pto.wait_flag["PIPE_V", "PIPE_MTE3", "EVENT_ID0"] + pto.mte_ub_gm %ub_out, %output, %c128_i64 + nburst(%c32_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64 + pto.mte_ub_gm %ub_spr_raw, %spr_output, %c128_i64 + nburst(%c32_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64 + pto.barrier #pto.pipe + return + } +} diff --git a/test/vpto/cases/micro-op/vector-load-store/soft-post-update-address-units/launch.cpp b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-address-units/launch.cpp new file mode 100644 index 0000000000..edb35b2937 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-address-units/launch.cpp @@ -0,0 +1,49 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#ifndef __VEC_SCOPE__ +#define __VEC_SCOPE__ +#endif + +#if defined(__CCE_AICORE__) && defined(__NPU_ARCH__) && (__NPU_ARCH__ == 2201) +typedef struct { unsigned char v; } hifloat8_t; +typedef struct { unsigned char v; } float8_e4m3_t; +typedef struct { unsigned char v; } float8_e5m2_t; +typedef struct { unsigned char v; } float8_e8m0_t; +typedef struct { unsigned char v; } float4_e1m2x2_t; +typedef struct { unsigned char v; } float4_e2m1x2_t; +#endif + +#include + +#if !defined(__CCE_AICORE__) && !defined(TMRGSORT_HPP) +struct MrgSortExecutedNumList { + uint16_t mrgSortList0; + uint16_t mrgSortList1; + uint16_t mrgSortList2; + uint16_t mrgSortList3; +}; +#endif + +#ifndef __CPU_SIM +#include "acl/acl.h" +#endif + +extern "C" __global__ [aicore] void +soft_post_update_address_units_kernel(__gm__ float *input, + __gm__ float *output, + __gm__ uint32_t *spr_input, + __gm__ uint32_t *spr_output); + +void LaunchSoftPostUpdateAddressUnits(float *input, float *output, + uint32_t *sprInput, uint32_t *sprOutput, + void *stream) { + soft_post_update_address_units_kernel<<<1, nullptr, stream>>>( + (__gm__ float *)input, (__gm__ float *)output, + (__gm__ uint32_t *)sprInput, (__gm__ uint32_t *)sprOutput); +} diff --git a/test/vpto/cases/micro-op/vector-load-store/soft-post-update-address-units/main.cpp b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-address-units/main.cpp new file mode 100644 index 0000000000..db8b1d48cc --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-address-units/main.cpp @@ -0,0 +1,134 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#include "acl/acl.h" +#include "test_common.h" + +#include +#include +#include + +using namespace PtoTestCommon; + +void LaunchSoftPostUpdateAddressUnits(float *input, float *output, + uint32_t *sprInput, uint32_t *sprOutput, + void *stream); + +namespace { +constexpr size_t kElementCount = 1024; +constexpr size_t kBufferSize = kElementCount * sizeof(float); +} + +#define ACL_CHECK(expr) \ + do { \ + const aclError ret = (expr); \ + if (ret != ACL_SUCCESS) { \ + std::fprintf(stderr, "[ERROR] %s failed: %d (%s:%d)\n", #expr, \ + static_cast(ret), __FILE__, __LINE__); \ + const char *recent = aclGetRecentErrMsg(); \ + if (recent != nullptr && recent[0] != '\0') \ + std::fprintf(stderr, "[ERROR] RecentErrMsg: %s\n", recent); \ + rc = 1; \ + goto cleanup; \ + } \ + } while (0) + +int main() { + float *inputHost = nullptr; + float *outputHost = nullptr; + uint32_t *sprInputHost = nullptr; + uint32_t *sprOutputHost = nullptr; + float *inputDevice = nullptr; + float *outputDevice = nullptr; + uint32_t *sprInputDevice = nullptr; + uint32_t *sprOutputDevice = nullptr; + aclrtStream stream = nullptr; + int rc = 0; + bool aclInited = false; + bool deviceSet = false; + int deviceId = 0; + size_t inputSize = kBufferSize; + size_t outputSize = kBufferSize; + size_t sprInputSize = kBufferSize; + size_t sprOutputSize = kBufferSize; + + ACL_CHECK(aclInit(nullptr)); + aclInited = true; + if (const char *envDevice = std::getenv("ACL_DEVICE_ID")) + deviceId = std::atoi(envDevice); + ACL_CHECK(aclrtSetDevice(deviceId)); + deviceSet = true; + ACL_CHECK(aclrtCreateStream(&stream)); + + ACL_CHECK(aclrtMallocHost(reinterpret_cast(&inputHost), kBufferSize)); + ACL_CHECK(aclrtMallocHost(reinterpret_cast(&outputHost), kBufferSize)); + ACL_CHECK( + aclrtMallocHost(reinterpret_cast(&sprInputHost), kBufferSize)); + ACL_CHECK( + aclrtMallocHost(reinterpret_cast(&sprOutputHost), kBufferSize)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&inputDevice), kBufferSize, + ACL_MEM_MALLOC_HUGE_FIRST)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&outputDevice), kBufferSize, + ACL_MEM_MALLOC_HUGE_FIRST)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&sprInputDevice), kBufferSize, + ACL_MEM_MALLOC_HUGE_FIRST)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&sprOutputDevice), + kBufferSize, ACL_MEM_MALLOC_HUGE_FIRST)); + + ReadFile("./input.bin", inputSize, inputHost, kBufferSize); + ReadFile("./output.bin", outputSize, outputHost, kBufferSize); + ReadFile("./input_spr.bin", sprInputSize, sprInputHost, kBufferSize); + ReadFile("./output_spr.bin", sprOutputSize, sprOutputHost, kBufferSize); + ACL_CHECK(aclrtMemcpy(inputDevice, kBufferSize, inputHost, kBufferSize, + ACL_MEMCPY_HOST_TO_DEVICE)); + ACL_CHECK(aclrtMemcpy(outputDevice, kBufferSize, outputHost, kBufferSize, + ACL_MEMCPY_HOST_TO_DEVICE)); + ACL_CHECK(aclrtMemcpy(sprInputDevice, kBufferSize, sprInputHost, kBufferSize, + ACL_MEMCPY_HOST_TO_DEVICE)); + ACL_CHECK(aclrtMemcpy(sprOutputDevice, kBufferSize, sprOutputHost, + kBufferSize, ACL_MEMCPY_HOST_TO_DEVICE)); + + LaunchSoftPostUpdateAddressUnits(inputDevice, outputDevice, sprInputDevice, + sprOutputDevice, stream); + ACL_CHECK(aclrtSynchronizeStream(stream)); + ACL_CHECK(aclrtMemcpy(outputHost, kBufferSize, outputDevice, kBufferSize, + ACL_MEMCPY_DEVICE_TO_HOST)); + ACL_CHECK(aclrtMemcpy(sprOutputHost, kBufferSize, sprOutputDevice, + kBufferSize, ACL_MEMCPY_DEVICE_TO_HOST)); + WriteFile("./output.bin", outputHost, kBufferSize); + WriteFile("./output_spr.bin", sprOutputHost, kBufferSize); + +cleanup: + aclrtFree(inputDevice); + aclrtFree(outputDevice); + aclrtFree(sprInputDevice); + aclrtFree(sprOutputDevice); + aclrtFreeHost(inputHost); + aclrtFreeHost(outputHost); + aclrtFreeHost(sprInputHost); + aclrtFreeHost(sprOutputHost); + if (stream != nullptr) { + const aclError ret = aclrtDestroyStream(stream); + if (ret != ACL_SUCCESS) + std::fprintf(stderr, "[ERROR] aclrtDestroyStream failed: %d\n", + static_cast(ret)); + } + if (deviceSet) { + const aclError ret = aclrtResetDevice(deviceId); + if (ret != ACL_SUCCESS) + std::fprintf(stderr, "[ERROR] aclrtResetDevice failed: %d\n", + static_cast(ret)); + } + if (aclInited) { + const aclError ret = aclFinalize(); + if (ret != ACL_SUCCESS) + std::fprintf(stderr, "[ERROR] aclFinalize failed: %d\n", + static_cast(ret)); + } + return rc; +} diff --git a/test/vpto/cases/micro-op/vector-load-store/soft-post-update-address-units/ptoas.flags b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-address-units/ptoas.flags new file mode 100644 index 0000000000..28c4691e94 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-address-units/ptoas.flags @@ -0,0 +1 @@ +--pto-arch a5 --pto-backend=vpto --enable-vpto-soft-postupdate diff --git a/test/vpto/cases/micro-op/vector-load-store/soft-post-update-step4-combined/compare.py b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-step4-combined/compare.py new file mode 100644 index 0000000000..349a69e386 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-step4-combined/compare.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import numpy as np + + +def main() -> None: + golden = np.fromfile("golden_output.bin", dtype=np.uint8) + output = np.fromfile("output.bin", dtype=np.uint8) + if golden.shape != output.shape or not np.array_equal(golden, output): + mismatch = np.flatnonzero(golden != output) + idx = int(mismatch[0]) if mismatch.size else 0 + print( + f"[ERROR] Step 4 combined mismatch: idx={idx}, " + f"golden={int(golden[idx]) if golden.size else 'n/a'}, " + f"output={int(output[idx]) if output.size else 'n/a'}" + ) + raise SystemExit(2) + print("[INFO] Step 4 combined soft-postupdate compare passed") + + +if __name__ == "__main__": + main() diff --git a/test/vpto/cases/micro-op/vector-load-store/soft-post-update-step4-combined/golden.py b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-step4-combined/golden.py new file mode 100644 index 0000000000..a30264acd5 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-step4-combined/golden.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import numpy as np + + +BUFFER_BYTES = 4096 +SENTINEL = 0xA5 + + +def main() -> None: + data_f32 = (np.arange(BUFFER_BYTES // 4, dtype=np.float32) * 0.25) + 1.0 + data = data_f32.view(np.uint8) + golden = np.full((BUFFER_BYTES,), SENTINEL, dtype=np.uint8) + + # vldsx2 DINTLV_B32: low receives even f32 elements, high receives odd. + vldsx2 = np.empty((256,), dtype=np.float32) + for iteration, base in enumerate((0, 128)): + dst = iteration * 128 + vldsx2[dst : dst + 64] = data_f32[base : base + 128 : 2] + vldsx2[dst + 64 : dst + 128] = data_f32[base + 1 : base + 128 : 2] + golden[0:1024] = vldsx2.view(np.uint8) + + # vsldb starts each access repeat_stride=8 blocks (64 f32 elements) past + # the current base, and the loop recurrence advances by that same amount. + vsldb_source = 2048 // 4 + vsldb = np.concatenate( + ( + data_f32[vsldb_source + 64 : vsldb_source + 128], + data_f32[vsldb_source + 128 : vsldb_source + 192], + ) + ) + golden[1024:1536] = vsldb.view(np.uint8) + + # Two NORM predicate images advance by 32 bytes each. + golden[2048:2112] = data[1024:1088] + + # Each sprsts writes one 32-bit SPR value. The returned-base recurrence is + # 32 bytes, so the two zero words are separated by 28 untouched bytes. + golden[3072:3076] = 0 + golden[3104:3108] = 0 + + data.tofile("input.bin") + np.full((BUFFER_BYTES,), SENTINEL, dtype=np.uint8).tofile("output.bin") + golden.tofile("golden_output.bin") + + +if __name__ == "__main__": + main() diff --git a/test/vpto/cases/micro-op/vector-load-store/soft-post-update-step4-combined/kernel.pto b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-step4-combined/kernel.pto new file mode 100644 index 0000000000..a2a9f66021 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-step4-combined/kernel.pto @@ -0,0 +1,131 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// case: micro-op/vector-load-store/soft-post-update-step4-combined +// family: vector-load-store +// target_ops: pto.plds, pto.sprsts, pto.vldsx2, pto.vsldb +// target_pass: vpto-soft-postupdate +// scenarios: normal-loop-recurrence, byte-element-block-units, strict-output + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @soft_post_update_step4_combined_kernel( + %input: !pto.ptr, + %output: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c8 = arith.constant 8 : index + %c32 = arith.constant 32 : index + %c64 = arith.constant 64 : index + %c128 = arith.constant 128 : index + %c256 = arith.constant 256 : index + %c0_i32 = arith.constant 0 : i32 + %c64_i32 = arith.constant 64 : i32 + %c1_i16 = arith.constant 1 : i16 + %c8_i16 = arith.constant 8 : i16 + %c0_i64 = arith.constant 0 : i64 + %c32_i64 = arith.constant 32 : i64 + %c128_i64 = arith.constant 128 : i64 + %c1024_i64 = arith.constant 1024 : i64 + %c2048_i64 = arith.constant 2048 : i64 + %c8192_i64 = arith.constant 8192 : i64 + %c9216_i64 = arith.constant 9216 : i64 + %c10240_i64 = arith.constant 10240 : i64 + %c11264_i64 = arith.constant 11264 : i64 + + %ub_input_bytes = pto.castptr %c0_i64 : i64 -> !pto.ptr + %ub_input_f32 = pto.castptr %c0_i64 : i64 -> !pto.ptr + %ub_plds_input = pto.castptr %c1024_i64 : i64 -> !pto.ptr + %ub_vsldb_input = pto.castptr %c2048_i64 : i64 -> !pto.ptr + %ub_output_bytes = pto.castptr %c8192_i64 : i64 -> !pto.ptr + %ub_vldsx2_output = pto.castptr %c8192_i64 : i64 -> !pto.ptr + %ub_vsldb_output = pto.castptr %c9216_i64 : i64 -> !pto.ptr + %ub_plds_output = pto.castptr %c10240_i64 : i64 -> !pto.ptr + %ub_sprsts_output = pto.castptr %c11264_i64 : i64 -> !pto.ptr + + pto.mte_gm_ub %input, %ub_input_bytes, %c0_i64, %c128_i64 + nburst(%c32_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64, i64 + pto.mte_gm_ub %output, %ub_output_bytes, %c0_i64, %c128_i64 + nburst(%c32_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64, i64 + + pto.set_flag["PIPE_MTE2", "PIPE_V", "EVENT_ID0"] + pto.wait_flag["PIPE_MTE2", "PIPE_V", "EVENT_ID0"] + + pto.vecscope { + %mask, %remaining = pto.plt_b32 %c64_i32 + : i32 -> !pto.mask, i32 + + // vldsx2 advances in f32 elements. The two normal iterations cover two + // consecutive 128-element interleaved source tiles. + scf.for %iv = %c0 to %c256 step %c128 { + %low, %high = pto.vldsx2 %ub_input_f32[%iv], "DINTLV_B32" + : !pto.ptr, index + -> !pto.vreg<64xf32>, !pto.vreg<64xf32> + %high_offset = arith.addi %iv, %c64 : index + pto.vsts %low, %ub_vldsx2_output[%iv], %mask {dist = "NORM_B32"} + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + pto.vsts %high, %ub_vldsx2_output[%high_offset], %mask {dist = "NORM_B32"} + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + + // repeat_stride=8 means eight 32-byte blocks. The software pointer + // recurrence is the equivalent 64 f32 elements. + scf.for %iv = %c0 to %c2 step %c1 + iter_args(%src = %ub_vsldb_input, %dst = %ub_vsldb_output) + -> (!pto.ptr, !pto.ptr) { + %value = pto.vsldb %src, %c1_i16, %c8_i16, %mask + : !pto.ptr, i16, i16, !pto.mask + -> !pto.vreg<64xf32> + pto.vsts %value, %dst[%c0], %mask {dist = "NORM_B32"} + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %next_src = pto.addptr %src, %c64 + : !pto.ptr -> !pto.ptr + %next_dst = pto.addptr %dst, %c64 + : !pto.ptr -> !pto.ptr + scf.yield %next_src, %next_dst + : !pto.ptr, !pto.ptr + } + + // plds/psts scalar offsets and sprsts scalar offsets are bytes. + scf.for %iv = %c0 to %c2 step %c1 + iter_args(%src = %ub_plds_input, %dst = %ub_plds_output) + -> (!pto.ptr, !pto.ptr) { + %predicate = pto.plds %src[%c0], "NORM" + : !pto.ptr, index -> !pto.mask + pto.mem_bar "VST_VLD" + pto.psts %predicate, %dst[%c0], "NORM" + : !pto.mask, !pto.ptr, index + %next_src = pto.addptr %src, %c32 + : !pto.ptr -> !pto.ptr + %next_dst = pto.addptr %dst, %c32 + : !pto.ptr -> !pto.ptr + scf.yield %next_src, %next_dst + : !pto.ptr, !pto.ptr + } + + scf.for %iv = %c0 to %c2 step %c1 + iter_args(%dst = %ub_sprsts_output) -> (!pto.ptr) { + pto.sprclr "AR" + pto.sprsts "AR", %dst[%c0_i32] : !pto.ptr, i32 + %next_dst = pto.addptr %dst, %c8 + : !pto.ptr -> !pto.ptr + scf.yield %next_dst : !pto.ptr + } + } + + pto.set_flag["PIPE_V", "PIPE_MTE3", "EVENT_ID0"] + pto.wait_flag["PIPE_V", "PIPE_MTE3", "EVENT_ID0"] + pto.mte_ub_gm %ub_output_bytes, %output, %c128_i64 + nburst(%c32_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64 + pto.barrier #pto.pipe + return + } +} diff --git a/test/vpto/cases/micro-op/vector-load-store/soft-post-update-step4-combined/launch.cpp b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-step4-combined/launch.cpp new file mode 100644 index 0000000000..d1d76de1c8 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-step4-combined/launch.cpp @@ -0,0 +1,45 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#ifndef __VEC_SCOPE__ +#define __VEC_SCOPE__ +#endif + +#if defined(__CCE_AICORE__) && defined(__NPU_ARCH__) && (__NPU_ARCH__ == 2201) +typedef struct { unsigned char v; } hifloat8_t; +typedef struct { unsigned char v; } float8_e4m3_t; +typedef struct { unsigned char v; } float8_e5m2_t; +typedef struct { unsigned char v; } float8_e8m0_t; +typedef struct { unsigned char v; } float4_e1m2x2_t; +typedef struct { unsigned char v; } float4_e2m1x2_t; +#endif + +#include + +#if !defined(__CCE_AICORE__) && !defined(TMRGSORT_HPP) +struct MrgSortExecutedNumList { + uint16_t mrgSortList0; + uint16_t mrgSortList1; + uint16_t mrgSortList2; + uint16_t mrgSortList3; +}; +#endif + +#ifndef __CPU_SIM +#include "acl/acl.h" +#endif + +extern "C" __global__ [aicore] void +soft_post_update_step4_combined_kernel(__gm__ uint8_t *input, + __gm__ uint8_t *output); + +void LaunchSoftPostUpdateStep4Combined(uint8_t *input, uint8_t *output, + void *stream) { + soft_post_update_step4_combined_kernel<<<1, nullptr, stream>>>( + (__gm__ uint8_t *)input, (__gm__ uint8_t *)output); +} diff --git a/test/vpto/cases/micro-op/vector-load-store/soft-post-update-step4-combined/main.cpp b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-step4-combined/main.cpp new file mode 100644 index 0000000000..b8536fb73c --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-step4-combined/main.cpp @@ -0,0 +1,91 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#include "acl/acl.h" +#include "test_common.h" + +#include +#include +#include + +using namespace PtoTestCommon; + +void LaunchSoftPostUpdateStep4Combined(uint8_t *input, uint8_t *output, + void *stream); + +namespace { +constexpr size_t kBufferBytes = 4096; +} // namespace + +#define ACL_CHECK(expr) \ + do { \ + const aclError ret = (expr); \ + if (ret != ACL_SUCCESS) { \ + std::fprintf(stderr, "[ERROR] %s failed: %d (%s:%d)\n", #expr, \ + static_cast(ret), __FILE__, __LINE__); \ + rc = 1; \ + goto cleanup; \ + } \ + } while (0) + +int main() { + uint8_t *inputHost = nullptr; + uint8_t *outputHost = nullptr; + uint8_t *inputDevice = nullptr; + uint8_t *outputDevice = nullptr; + aclrtStream stream = nullptr; + int rc = 0; + bool aclInited = false; + bool deviceSet = false; + int deviceId = 0; + size_t inputSize = kBufferBytes; + size_t outputSize = kBufferBytes; + + ACL_CHECK(aclInit(nullptr)); + aclInited = true; + if (const char *envDevice = std::getenv("ACL_DEVICE_ID")) + deviceId = std::atoi(envDevice); + ACL_CHECK(aclrtSetDevice(deviceId)); + deviceSet = true; + ACL_CHECK(aclrtCreateStream(&stream)); + + ACL_CHECK( + aclrtMallocHost(reinterpret_cast(&inputHost), kBufferBytes)); + ACL_CHECK( + aclrtMallocHost(reinterpret_cast(&outputHost), kBufferBytes)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&inputDevice), kBufferBytes, + ACL_MEM_MALLOC_HUGE_FIRST)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&outputDevice), kBufferBytes, + ACL_MEM_MALLOC_HUGE_FIRST)); + + ReadFile("./input.bin", inputSize, inputHost, kBufferBytes); + ReadFile("./output.bin", outputSize, outputHost, kBufferBytes); + ACL_CHECK(aclrtMemcpy(inputDevice, kBufferBytes, inputHost, kBufferBytes, + ACL_MEMCPY_HOST_TO_DEVICE)); + ACL_CHECK(aclrtMemcpy(outputDevice, kBufferBytes, outputHost, kBufferBytes, + ACL_MEMCPY_HOST_TO_DEVICE)); + + LaunchSoftPostUpdateStep4Combined(inputDevice, outputDevice, stream); + ACL_CHECK(aclrtSynchronizeStream(stream)); + ACL_CHECK(aclrtMemcpy(outputHost, kBufferBytes, outputDevice, kBufferBytes, + ACL_MEMCPY_DEVICE_TO_HOST)); + WriteFile("./output.bin", outputHost, kBufferBytes); + +cleanup: + aclrtFree(inputDevice); + aclrtFree(outputDevice); + aclrtFreeHost(inputHost); + aclrtFreeHost(outputHost); + if (stream != nullptr) + aclrtDestroyStream(stream); + if (deviceSet) + aclrtResetDevice(deviceId); + if (aclInited) + aclFinalize(); + return rc; +} diff --git a/test/vpto/cases/micro-op/vector-load-store/soft-post-update-step4-combined/ptoas.flags b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-step4-combined/ptoas.flags new file mode 100644 index 0000000000..28c4691e94 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-step4-combined/ptoas.flags @@ -0,0 +1 @@ +--pto-arch a5 --pto-backend=vpto --enable-vpto-soft-postupdate diff --git a/test/vpto/cases/micro-op/vector-load-store/soft-post-update-vldus-vstus/compare.py b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-vldus-vstus/compare.py new file mode 100644 index 0000000000..f9b3ba3c04 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-vldus-vstus/compare.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import numpy as np + + +LOAD_BEGIN = 0 +LOAD_END = 64 +LOAD_INPUT_BEGIN = 128 +STORE_BEGIN = 256 + + +def fail(message: str) -> None: + print(f"[ERROR] {message}") + raise SystemExit(2) + + +def main() -> None: + input_data = np.fromfile("input.bin", dtype=np.float32) + initial = np.fromfile("initial.bin", dtype=np.float32) + explicit = np.fromfile("explicit_output.bin", dtype=np.float32) + rewritten = np.fromfile("rewritten_output.bin", dtype=np.float32) + + if not (input_data.shape == initial.shape == explicit.shape == rewritten.shape): + fail( + "shape mismatch: " + f"input={input_data.shape}, initial={initial.shape}, " + f"explicit={explicit.shape}, rewritten={rewritten.shape}" + ) + + mismatch = np.flatnonzero(explicit != rewritten) + if mismatch.size: + idx = int(mismatch[0]) + fail( + "explicit and rewritten post-update paths differ at " + f"index {idx}: explicit={explicit[idx]}, rewritten={rewritten[idx]}" + ) + + expected_load = input_data[LOAD_INPUT_BEGIN : LOAD_INPUT_BEGIN + LOAD_END] + if not np.array_equal(explicit[LOAD_BEGIN:LOAD_END], expected_load): + mismatch = np.flatnonzero( + explicit[LOAD_BEGIN:LOAD_END] != expected_load + ) + idx = int(mismatch[0]) if mismatch.size else LOAD_BEGIN + fail( + f"vldus load result mismatch at index {idx}: " + f"expected={expected_load[idx]}, output={explicit[idx]}" + ) + + store_changes = np.flatnonzero( + explicit[STORE_BEGIN:] != initial[STORE_BEGIN:] + ) + if not store_changes.size: + fail("vstus/vstar did not change the store probe region") + if int(store_changes[0]) != 0: + fail( + "vstus first write was pre-offset: first changed store-relative " + f"index is {int(store_changes[0])}, expected 0" + ) + + print( + "[INFO] compare passed: explicit and rewritten paths match; " + f"vstus first changed store-relative index={int(store_changes[0])}" + ) + + +if __name__ == "__main__": + main() diff --git a/test/vpto/cases/micro-op/vector-load-store/soft-post-update-vldus-vstus/golden.py b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-vldus-vstus/golden.py new file mode 100644 index 0000000000..87464dd721 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-vldus-vstus/golden.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import argparse +from pathlib import Path + +import numpy as np + + +ELEMENTS = 1024 + + +def generate(output_dir: Path) -> None: + input_data = (np.arange(ELEMENTS, dtype=np.float32) + 1024.25).astype( + np.float32 + ) + initial = (-16384.0 - np.arange(ELEMENTS, dtype=np.float32)).astype( + np.float32 + ) + + output_dir.mkdir(parents=True, exist_ok=True) + input_data.tofile(output_dir / "input.bin") + initial.tofile(output_dir / "initial.bin") + initial.tofile(output_dir / "explicit_output.bin") + initial.tofile(output_dir / "rewritten_output.bin") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Generate vldus/vstus soft post-update SIM probe data." + ) + parser.add_argument("--output-dir", type=Path, default=Path(".")) + args = parser.parse_args() + generate(args.output_dir) + + +if __name__ == "__main__": + main() diff --git a/test/vpto/cases/micro-op/vector-load-store/soft-post-update-vldus-vstus/kernel.pto b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-vldus-vstus/kernel.pto new file mode 100644 index 0000000000..e1a4dce25a --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-vldus-vstus/kernel.pto @@ -0,0 +1,156 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// case: micro-op/vector-load-store/soft-post-update-vldus-vstus +// family: vector-load-store +// target_pass: vpto-soft-postupdate +// target_ops: pto.vldus, pto.vstus +// scenarios: explicit-vs-rewritten-post-update, vstus-first-base + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @soft_post_update_vldus_vstus_kernel( + %input: !pto.ptr, + %initial: !pto.ptr, + %explicit_output: !pto.ptr, + %rewritten_output: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : index + %c64 = arith.constant 64 : index + %c128 = arith.constant 128 : index + %c192 = arith.constant 192 : index + %c256 = arith.constant 256 : index + %c3_i32 = arith.constant 3 : i32 + %c64_i32 = arith.constant 64 : i32 + %c0_i64 = arith.constant 0 : i64 + %c32_i64 = arith.constant 32 : i64 + %c128_i64 = arith.constant 128 : i64 + %c4096_i64 = arith.constant 4096 : i64 + %c8192_i64 = arith.constant 8192 : i64 + + %ub_input = pto.castptr %c0_i64 : i64 -> !pto.ptr + %ub_explicit = pto.castptr %c4096_i64 : i64 -> !pto.ptr + %ub_rewritten = pto.castptr %c8192_i64 : i64 -> !pto.ptr + %ub_explicit_store = pto.addptr %ub_explicit, %c256 + : !pto.ptr -> !pto.ptr + %ub_rewritten_store = pto.addptr %ub_rewritten, %c256 + : !pto.ptr -> !pto.ptr + + pto.mte_gm_ub %input, %ub_input, %c0_i64, %c128_i64 + nburst(%c32_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64, i64 + pto.mte_gm_ub %initial, %ub_explicit, %c0_i64, %c128_i64 + nburst(%c32_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64, i64 + pto.mte_gm_ub %initial, %ub_rewritten, %c0_i64, %c128_i64 + nburst(%c32_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64, i64 + + pto.set_flag["PIPE_MTE2", "PIPE_V", "EVENT_ID0"] + pto.wait_flag["PIPE_MTE2", "PIPE_V", "EVENT_ID0"] + + pto.vecscope { + %mask, %remaining = pto.plt_b32 %c64_i32 + : i32 -> !pto.mask, i32 + + // Explicit post-update reference for vldus. + %explicit_load_align = pto.vldas %ub_input + : !pto.ptr -> !pto.align + %explicit_value0, %explicit_load_align1, %explicit_input1 = + pto.vldus %ub_input, %explicit_load_align, %c64 + : !pto.ptr, !pto.align, index + -> !pto.vreg<64xf32>, !pto.align, !pto.ptr + %explicit_value1, %explicit_load_align2, %explicit_input2 = + pto.vldus %explicit_input1, %explicit_load_align1, %c64 + : !pto.ptr, !pto.align, index + -> !pto.vreg<64xf32>, !pto.align, !pto.ptr + %explicit_value2, %explicit_load_align3, %explicit_input3 = + pto.vldus %explicit_input2, %explicit_load_align2, %c64 + : !pto.ptr, !pto.align, index + -> !pto.vreg<64xf32>, !pto.align, !pto.ptr + // Keep only the final loaded vector. Using a fixed destination keeps + // this observation store outside the post-update transformation. + pto.vsts %explicit_value2, %ub_explicit[%c0], %mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + + // The pass must rewrite this recurrence to the same vldus post form. + %rewritten_load_align = pto.vldas %ub_input + : !pto.ptr -> !pto.align + %final_load_ptr, %final_load_align = + scf.for %iv = %c0 to %c192 step %c64 + iter_args(%load_ptr = %ub_input, + %load_align = %rewritten_load_align) + -> (!pto.ptr, !pto.align) { + %value, %next_load_align = pto.vldus %load_ptr, %load_align + : !pto.ptr, !pto.align + -> !pto.vreg<64xf32>, !pto.align + pto.vsts %value, %ub_rewritten[%c0], %mask + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %next_load_ptr = pto.addptr %load_ptr, %c64 + : !pto.ptr -> !pto.ptr + scf.yield %next_load_ptr, %next_load_align + : !pto.ptr, !pto.align + } + + %store_value = pto.vlds %ub_input[%c0] + : !pto.ptr -> !pto.vreg<64xf32> + + // Explicit post-update reference for vstus. The returned pointer is the + // logical flush point after advancing by three elements. + %explicit_store_align = pto.init_align : !pto.align + %explicit_store_align1, %explicit_store1 = + pto.vstus %explicit_store_align, %c3_i32, %store_value, + %ub_explicit_store + : !pto.align, i32, !pto.vreg<64xf32>, !pto.ptr + -> !pto.align, !pto.ptr + %explicit_store_align2, %explicit_store2 = + pto.vstus %explicit_store_align1, %c3_i32, %store_value, + %explicit_store1 + : !pto.align, i32, !pto.vreg<64xf32>, !pto.ptr + -> !pto.align, !pto.ptr + %explicit_store_align3, %explicit_store3 = + pto.vstus %explicit_store_align2, %c3_i32, %store_value, + %explicit_store2 + : !pto.align, i32, !pto.vreg<64xf32>, !pto.ptr + -> !pto.align, !pto.ptr + pto.vstar %explicit_store_align3, %explicit_store3 + : !pto.align, !pto.ptr + + // The offset is the post-access stream advance. The pass must keep the + // first vstus base at %ub_rewritten_store, not pre-offset it by three. + %rewritten_store_align = pto.init_align : !pto.align + %final_store_ptr, %final_store_align = + scf.for %iv = %c0 to %c3 step %c1 + iter_args(%store_ptr = %ub_rewritten_store, + %store_align = %rewritten_store_align) + -> (!pto.ptr, !pto.align) { + %next_store_align = pto.vstus %store_align, %c3_i32, + %store_value, %store_ptr + : !pto.align, i32, !pto.vreg<64xf32>, !pto.ptr + -> !pto.align + %next_store_ptr = pto.addptr %store_ptr, %c3 + : !pto.ptr -> !pto.ptr + scf.yield %next_store_ptr, %next_store_align + : !pto.ptr, !pto.align + } + pto.vstar %final_store_align, %final_store_ptr + : !pto.align, !pto.ptr + } + + pto.set_flag["PIPE_V", "PIPE_MTE3", "EVENT_ID0"] + pto.wait_flag["PIPE_V", "PIPE_MTE3", "EVENT_ID0"] + pto.mte_ub_gm %ub_explicit, %explicit_output, %c128_i64 + nburst(%c32_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64 + pto.mte_ub_gm %ub_rewritten, %rewritten_output, %c128_i64 + nburst(%c32_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64 + pto.barrier #pto.pipe + return + } +} diff --git a/test/vpto/cases/micro-op/vector-load-store/soft-post-update-vldus-vstus/launch.cpp b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-vldus-vstus/launch.cpp new file mode 100644 index 0000000000..341b3c8f63 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-vldus-vstus/launch.cpp @@ -0,0 +1,49 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#ifndef __VEC_SCOPE__ +#define __VEC_SCOPE__ +#endif + +#if defined(__CCE_AICORE__) && defined(__NPU_ARCH__) && (__NPU_ARCH__ == 2201) +typedef struct { unsigned char v; } hifloat8_t; +typedef struct { unsigned char v; } float8_e4m3_t; +typedef struct { unsigned char v; } float8_e5m2_t; +typedef struct { unsigned char v; } float8_e8m0_t; +typedef struct { unsigned char v; } float4_e1m2x2_t; +typedef struct { unsigned char v; } float4_e2m1x2_t; +#endif + +#include + +#if !defined(__CCE_AICORE__) && !defined(TMRGSORT_HPP) +struct MrgSortExecutedNumList { + uint16_t mrgSortList0; + uint16_t mrgSortList1; + uint16_t mrgSortList2; + uint16_t mrgSortList3; +}; +#endif + +#ifndef __CPU_SIM +#include "acl/acl.h" +#endif + +extern "C" __global__ [aicore] void +soft_post_update_vldus_vstus_kernel(__gm__ float *input, + __gm__ float *initial, + __gm__ float *explicit_output, + __gm__ float *rewritten_output); + +void LaunchSoftPostUpdateVldusVstus(float *input, float *initial, + float *explicitOutput, + float *rewrittenOutput, void *stream) { + soft_post_update_vldus_vstus_kernel<<<1, nullptr, stream>>>( + (__gm__ float *)input, (__gm__ float *)initial, + (__gm__ float *)explicitOutput, (__gm__ float *)rewrittenOutput); +} diff --git a/test/vpto/cases/micro-op/vector-load-store/soft-post-update-vldus-vstus/main.cpp b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-vldus-vstus/main.cpp new file mode 100644 index 0000000000..6b8be805ec --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-vldus-vstus/main.cpp @@ -0,0 +1,135 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#include "acl/acl.h" +#include "test_common.h" + +#include +#include + +using namespace PtoTestCommon; + +void LaunchSoftPostUpdateVldusVstus(float *input, float *initial, + float *explicitOutput, + float *rewrittenOutput, void *stream); + +namespace { +constexpr size_t kElementCount = 1024; +constexpr size_t kBufferSize = kElementCount * sizeof(float); +} + +#define ACL_CHECK(expr) \ + do { \ + const aclError ret = (expr); \ + if (ret != ACL_SUCCESS) { \ + std::fprintf(stderr, "[ERROR] %s failed: %d (%s:%d)\n", #expr, \ + static_cast(ret), __FILE__, __LINE__); \ + const char *recent = aclGetRecentErrMsg(); \ + if (recent != nullptr && recent[0] != '\0') \ + std::fprintf(stderr, "[ERROR] RecentErrMsg: %s\n", recent); \ + rc = 1; \ + goto cleanup; \ + } \ + } while (0) + +int main() { + float *inputHost = nullptr; + float *initialHost = nullptr; + float *explicitHost = nullptr; + float *rewrittenHost = nullptr; + float *inputDevice = nullptr; + float *initialDevice = nullptr; + float *explicitDevice = nullptr; + float *rewrittenDevice = nullptr; + aclrtStream stream = nullptr; + int rc = 0; + bool aclInited = false; + bool deviceSet = false; + int deviceId = 0; + size_t inputSize = kBufferSize; + size_t initialSize = kBufferSize; + size_t explicitSize = kBufferSize; + size_t rewrittenSize = kBufferSize; + + ACL_CHECK(aclInit(nullptr)); + aclInited = true; + if (const char *envDevice = std::getenv("ACL_DEVICE_ID")) + deviceId = std::atoi(envDevice); + ACL_CHECK(aclrtSetDevice(deviceId)); + deviceSet = true; + ACL_CHECK(aclrtCreateStream(&stream)); + + ACL_CHECK(aclrtMallocHost(reinterpret_cast(&inputHost), kBufferSize)); + ACL_CHECK( + aclrtMallocHost(reinterpret_cast(&initialHost), kBufferSize)); + ACL_CHECK( + aclrtMallocHost(reinterpret_cast(&explicitHost), kBufferSize)); + ACL_CHECK( + aclrtMallocHost(reinterpret_cast(&rewrittenHost), kBufferSize)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&inputDevice), kBufferSize, + ACL_MEM_MALLOC_HUGE_FIRST)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&initialDevice), kBufferSize, + ACL_MEM_MALLOC_HUGE_FIRST)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&explicitDevice), kBufferSize, + ACL_MEM_MALLOC_HUGE_FIRST)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&rewrittenDevice), kBufferSize, + ACL_MEM_MALLOC_HUGE_FIRST)); + + ReadFile("./input.bin", inputSize, inputHost, kBufferSize); + ReadFile("./initial.bin", initialSize, initialHost, kBufferSize); + ReadFile("./explicit_output.bin", explicitSize, explicitHost, kBufferSize); + ReadFile("./rewritten_output.bin", rewrittenSize, rewrittenHost, + kBufferSize); + ACL_CHECK(aclrtMemcpy(inputDevice, kBufferSize, inputHost, kBufferSize, + ACL_MEMCPY_HOST_TO_DEVICE)); + ACL_CHECK(aclrtMemcpy(initialDevice, kBufferSize, initialHost, kBufferSize, + ACL_MEMCPY_HOST_TO_DEVICE)); + ACL_CHECK(aclrtMemcpy(explicitDevice, kBufferSize, explicitHost, kBufferSize, + ACL_MEMCPY_HOST_TO_DEVICE)); + ACL_CHECK(aclrtMemcpy(rewrittenDevice, kBufferSize, rewrittenHost, + kBufferSize, ACL_MEMCPY_HOST_TO_DEVICE)); + + LaunchSoftPostUpdateVldusVstus(inputDevice, initialDevice, explicitDevice, + rewrittenDevice, stream); + ACL_CHECK(aclrtSynchronizeStream(stream)); + ACL_CHECK(aclrtMemcpy(explicitHost, kBufferSize, explicitDevice, kBufferSize, + ACL_MEMCPY_DEVICE_TO_HOST)); + ACL_CHECK(aclrtMemcpy(rewrittenHost, kBufferSize, rewrittenDevice, + kBufferSize, ACL_MEMCPY_DEVICE_TO_HOST)); + WriteFile("./explicit_output.bin", explicitHost, kBufferSize); + WriteFile("./rewritten_output.bin", rewrittenHost, kBufferSize); + +cleanup: + aclrtFree(inputDevice); + aclrtFree(initialDevice); + aclrtFree(explicitDevice); + aclrtFree(rewrittenDevice); + aclrtFreeHost(inputHost); + aclrtFreeHost(initialHost); + aclrtFreeHost(explicitHost); + aclrtFreeHost(rewrittenHost); + if (stream != nullptr) { + const aclError ret = aclrtDestroyStream(stream); + if (ret != ACL_SUCCESS) + std::fprintf(stderr, "[ERROR] aclrtDestroyStream failed: %d\n", + static_cast(ret)); + } + if (deviceSet) { + const aclError ret = aclrtResetDevice(deviceId); + if (ret != ACL_SUCCESS) + std::fprintf(stderr, "[ERROR] aclrtResetDevice failed: %d\n", + static_cast(ret)); + } + if (aclInited) { + const aclError ret = aclFinalize(); + if (ret != ACL_SUCCESS) + std::fprintf(stderr, "[ERROR] aclFinalize failed: %d\n", + static_cast(ret)); + } + return rc; +} diff --git a/test/vpto/cases/micro-op/vector-load-store/soft-post-update-vldus-vstus/ptoas.flags b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-vldus-vstus/ptoas.flags new file mode 100644 index 0000000000..28c4691e94 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/soft-post-update-vldus-vstus/ptoas.flags @@ -0,0 +1 @@ +--pto-arch a5 --pto-backend=vpto --enable-vpto-soft-postupdate diff --git a/test/vpto/cases/micro-op/vector-load-store/spr-store-post-update/compare.py b/test/vpto/cases/micro-op/vector-load-store/spr-store-post-update/compare.py new file mode 100644 index 0000000000..d786f31000 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/spr-store-post-update/compare.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import numpy as np + + +def check(name: str) -> None: + golden = np.fromfile(f"golden_output_{name}.bin", dtype=np.uint32) + output = np.fromfile(f"output_{name}.bin", dtype=np.uint32) + if not np.array_equal(golden, output): + mismatch = np.flatnonzero(golden != output) + idx = int(mismatch[0]) if mismatch.size else 0 + print( + f"[ERROR] {name} mismatch: idx={idx}, " + f"golden={int(golden[idx]) if golden.size else 'n/a'}, " + f"output={int(output[idx]) if output.size else 'n/a'}" + ) + raise SystemExit(2) + + +def main() -> None: + check("i") + check("s") + print("[INFO] compare passed") + + +if __name__ == "__main__": + main() diff --git a/test/vpto/cases/micro-op/vector-load-store/spr-store-post-update/golden.py b/test/vpto/cases/micro-op/vector-load-store/spr-store-post-update/golden.py new file mode 100644 index 0000000000..ecc8284fac --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/spr-store-post-update/golden.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import numpy as np + + +def main() -> None: + data = (np.arange(64, dtype=np.uint32) * 17) + 3 + output_i = np.zeros((72,), dtype=np.uint32) + output_s = np.zeros((72,), dtype=np.uint32) + golden_i = output_i.copy() + golden_s = output_s.copy() + golden_i[8:72] = data + golden_s[8:72] = data + + data.tofile("input.bin") + output_i.tofile("output_i.bin") + output_s.tofile("output_s.bin") + golden_i.tofile("golden_output_i.bin") + golden_s.tofile("golden_output_s.bin") + + +if __name__ == "__main__": + main() diff --git a/test/vpto/cases/micro-op/vector-load-store/spr-store-post-update/kernel.pto b/test/vpto/cases/micro-op/vector-load-store/spr-store-post-update/kernel.pto new file mode 100644 index 0000000000..86b527ea4b --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/spr-store-post-update/kernel.pto @@ -0,0 +1,82 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// case: micro-op/vector-load-store/spr-store-post-update +// family: vector-load-store +// target_ops: pto.sprsti, pto.sprsts +// scenarios: post-update-result, immediate-word-stride, scalar-byte-stride + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @spr_store_post_update_kernel( + %input: !pto.ptr, + %output_i: !pto.ptr, + %output_s: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c0_i32 = arith.constant 0 : i32 + %c8_i32 = arith.constant 8 : i32 + %c32_i32 = arith.constant 32 : i32 + %c64_i32 = arith.constant 64 : i32 + %c0_i64 = arith.constant 0 : i64 + %c8_i64 = arith.constant 8 : i64 + %c9_i64 = arith.constant 9 : i64 + %c32_i64 = arith.constant 32 : i64 + %c4096_i64 = arith.constant 4096 : i64 + %c8192_i64 = arith.constant 8192 : i64 + + %ub_in = pto.castptr %c0_i64 : i64 -> !pto.ptr + %ub_out_i = pto.castptr %c4096_i64 : i64 -> !pto.ptr + %ub_out_s = pto.castptr %c8192_i64 : i64 -> !pto.ptr + + pto.mte_gm_ub %input, %ub_in, %c0_i64, %c32_i64 + nburst(%c8_i64, %c32_i64, %c32_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64, i64 + pto.mte_gm_ub %output_i, %ub_out_i, %c0_i64, %c32_i64 + nburst(%c9_i64, %c32_i64, %c32_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64, i64 + pto.mte_gm_ub %output_s, %ub_out_s, %c0_i64, %c32_i64 + nburst(%c9_i64, %c32_i64, %c32_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64, i64 + + pto.set_flag["PIPE_MTE2", "PIPE_V", "EVENT_ID0"] + pto.wait_flag["PIPE_MTE2", "PIPE_V", "EVENT_ID0"] + + pto.vecscope { + %mask, %remaining = pto.plt_b32 %c64_i32 + : i32 -> !pto.mask, i32 + %value = pto.vlds %ub_in[%c0] + : !pto.ptr -> !pto.vreg<64xui32> + + pto.sprclr "AR" + pto.sprsti "AR", %ub_out_i[%c0_i32] : !pto.ptr, i32 + pto.sprclr "AR" + %next_i = pto.sprsti "AR", %ub_out_i[%c8_i32] + : !pto.ptr, i32 -> !pto.ptr + pto.vsts %value, %next_i[%c0], %mask {dist = "NORM_B32"} + : !pto.vreg<64xui32>, !pto.ptr, !pto.mask + + pto.sprclr "AR" + pto.sprsts "AR", %ub_out_s[%c0_i32] : !pto.ptr, i32 + pto.sprclr "AR" + %next_s = pto.sprsts "AR", %ub_out_s[%c32_i32] + : !pto.ptr, i32 -> !pto.ptr + pto.vsts %value, %next_s[%c0], %mask {dist = "NORM_B32"} + : !pto.vreg<64xui32>, !pto.ptr, !pto.mask + } + + pto.set_flag["PIPE_V", "PIPE_MTE3", "EVENT_ID0"] + pto.wait_flag["PIPE_V", "PIPE_MTE3", "EVENT_ID0"] + pto.mte_ub_gm %ub_out_i, %output_i, %c32_i64 + nburst(%c9_i64, %c32_i64, %c32_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64 + pto.mte_ub_gm %ub_out_s, %output_s, %c32_i64 + nburst(%c9_i64, %c32_i64, %c32_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64 + pto.barrier #pto.pipe + return + } +} diff --git a/test/vpto/cases/micro-op/vector-load-store/spr-store-post-update/launch.cpp b/test/vpto/cases/micro-op/vector-load-store/spr-store-post-update/launch.cpp new file mode 100644 index 0000000000..2e6f862c26 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/spr-store-post-update/launch.cpp @@ -0,0 +1,29 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#ifndef __VEC_SCOPE__ +#define __VEC_SCOPE__ +#endif + +#include + +#ifndef __CPU_SIM +#include "acl/acl.h" +#endif + +extern "C" __global__[aicore] void +spr_store_post_update_kernel(__gm__ unsigned int *input, + __gm__ unsigned int *output_i, + __gm__ unsigned int *output_s); + +void LaunchSprStorePostUpdate(uint32_t *input, uint32_t *outputI, + uint32_t *outputS, void *stream) { + spr_store_post_update_kernel<<<1, nullptr, stream>>>( + (__gm__ unsigned int *)input, (__gm__ unsigned int *)outputI, + (__gm__ unsigned int *)outputS); +} diff --git a/test/vpto/cases/micro-op/vector-load-store/spr-store-post-update/main.cpp b/test/vpto/cases/micro-op/vector-load-store/spr-store-post-update/main.cpp new file mode 100644 index 0000000000..f693499823 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/spr-store-post-update/main.cpp @@ -0,0 +1,109 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#include "acl/acl.h" +#include "test_common.h" + +#include +#include +#include + +using namespace PtoTestCommon; + +void LaunchSprStorePostUpdate(uint32_t *input, uint32_t *outputI, + uint32_t *outputS, void *stream); + +namespace { +constexpr size_t kInputElements = 64; +constexpr size_t kOutputElements = 72; +constexpr size_t kInputBytes = kInputElements * sizeof(uint32_t); +constexpr size_t kOutputBytes = kOutputElements * sizeof(uint32_t); +} // namespace + +#define ACL_CHECK(expr) \ + do { \ + const aclError ret = (expr); \ + if (ret != ACL_SUCCESS) { \ + std::fprintf(stderr, "[ERROR] %s failed: %d (%s:%d)\n", #expr, \ + static_cast(ret), __FILE__, __LINE__); \ + rc = 1; \ + goto cleanup; \ + } \ + } while (0) + +int main() { + uint32_t *inputHost = nullptr; + uint32_t *outputIHost = nullptr; + uint32_t *outputSHost = nullptr; + uint32_t *inputDevice = nullptr; + uint32_t *outputIDevice = nullptr; + uint32_t *outputSDevice = nullptr; + aclrtStream stream = nullptr; + int rc = 0; + bool aclInited = false; + bool deviceSet = false; + int deviceId = 0; + size_t inputSize = kInputBytes; + size_t outputISize = kOutputBytes; + size_t outputSSize = kOutputBytes; + + ACL_CHECK(aclInit(nullptr)); + aclInited = true; + if (const char *envDevice = std::getenv("ACL_DEVICE_ID")) + deviceId = std::atoi(envDevice); + ACL_CHECK(aclrtSetDevice(deviceId)); + deviceSet = true; + ACL_CHECK(aclrtCreateStream(&stream)); + + ACL_CHECK( + aclrtMallocHost(reinterpret_cast(&inputHost), kInputBytes)); + ACL_CHECK( + aclrtMallocHost(reinterpret_cast(&outputIHost), kOutputBytes)); + ACL_CHECK( + aclrtMallocHost(reinterpret_cast(&outputSHost), kOutputBytes)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&inputDevice), kInputBytes, + ACL_MEM_MALLOC_HUGE_FIRST)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&outputIDevice), kOutputBytes, + ACL_MEM_MALLOC_HUGE_FIRST)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&outputSDevice), kOutputBytes, + ACL_MEM_MALLOC_HUGE_FIRST)); + + ReadFile("./input.bin", inputSize, inputHost, kInputBytes); + ReadFile("./output_i.bin", outputISize, outputIHost, kOutputBytes); + ReadFile("./output_s.bin", outputSSize, outputSHost, kOutputBytes); + ACL_CHECK(aclrtMemcpy(inputDevice, kInputBytes, inputHost, kInputBytes, + ACL_MEMCPY_HOST_TO_DEVICE)); + ACL_CHECK(aclrtMemcpy(outputIDevice, kOutputBytes, outputIHost, kOutputBytes, + ACL_MEMCPY_HOST_TO_DEVICE)); + ACL_CHECK(aclrtMemcpy(outputSDevice, kOutputBytes, outputSHost, kOutputBytes, + ACL_MEMCPY_HOST_TO_DEVICE)); + + LaunchSprStorePostUpdate(inputDevice, outputIDevice, outputSDevice, stream); + ACL_CHECK(aclrtSynchronizeStream(stream)); + ACL_CHECK(aclrtMemcpy(outputIHost, kOutputBytes, outputIDevice, kOutputBytes, + ACL_MEMCPY_DEVICE_TO_HOST)); + ACL_CHECK(aclrtMemcpy(outputSHost, kOutputBytes, outputSDevice, kOutputBytes, + ACL_MEMCPY_DEVICE_TO_HOST)); + WriteFile("./output_i.bin", outputIHost, kOutputBytes); + WriteFile("./output_s.bin", outputSHost, kOutputBytes); + +cleanup: + aclrtFree(inputDevice); + aclrtFree(outputIDevice); + aclrtFree(outputSDevice); + aclrtFreeHost(inputHost); + aclrtFreeHost(outputIHost); + aclrtFreeHost(outputSHost); + if (stream != nullptr) + aclrtDestroyStream(stream); + if (deviceSet) + aclrtResetDevice(deviceId); + if (aclInited) + aclFinalize(); + return rc; +} diff --git a/test/vpto/cases/micro-op/vector-load-store/vldsx2-vsldb-post-update/compare.py b/test/vpto/cases/micro-op/vector-load-store/vldsx2-vsldb-post-update/compare.py new file mode 100644 index 0000000000..a9dcdd29ba --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/vldsx2-vsldb-post-update/compare.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import numpy as np + + +def main() -> None: + golden = np.fromfile("golden_output.bin", dtype=np.float32) + output = np.fromfile("output.bin", dtype=np.float32) + if golden.shape != output.shape or not np.array_equal(golden, output): + mismatch = np.flatnonzero(golden != output) + idx = int(mismatch[0]) if mismatch.size else 0 + print( + f"[ERROR] mismatch: shape={output.shape}, idx={idx}, " + f"golden={golden[idx] if golden.size else 'n/a'}, " + f"output={output[idx] if output.size else 'n/a'}" + ) + raise SystemExit(2) + print("[INFO] compare passed") + + +if __name__ == "__main__": + main() diff --git a/test/vpto/cases/micro-op/vector-load-store/vldsx2-vsldb-post-update/golden.py b/test/vpto/cases/micro-op/vector-load-store/vldsx2-vsldb-post-update/golden.py new file mode 100644 index 0000000000..cc97c95ac4 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/vldsx2-vsldb-post-update/golden.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import numpy as np + + +def main() -> None: + data = (np.arange(512, dtype=np.float32) * 0.25) + 1.0 + golden = np.concatenate((data[128:192], data[64:128])) + data.tofile("input.bin") + np.zeros((128,), dtype=np.float32).tofile("output.bin") + golden.tofile("golden_output.bin") + + +if __name__ == "__main__": + main() diff --git a/test/vpto/cases/micro-op/vector-load-store/vldsx2-vsldb-post-update/kernel.pto b/test/vpto/cases/micro-op/vector-load-store/vldsx2-vsldb-post-update/kernel.pto new file mode 100644 index 0000000000..4cc704f16e --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/vldsx2-vsldb-post-update/kernel.pto @@ -0,0 +1,67 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// case: micro-op/vector-load-store/vldsx2-vsldb-post-update +// family: vector-load-store +// target_ops: pto.vldsx2, pto.vsldb +// scenarios: post-update-result, element-stride, block-stride + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @vldsx2_vsldb_post_update_kernel( + %input: !pto.ptr, %output: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c64 = arith.constant 64 : index + %c128 = arith.constant 128 : index + %c64_i32 = arith.constant 64 : i32 + %c1_i16 = arith.constant 1 : i16 + %c8_i16 = arith.constant 8 : i16 + %c0_i64 = arith.constant 0 : i64 + %c4_i64 = arith.constant 4 : i64 + %c16_i64 = arith.constant 16 : i64 + %c128_i64 = arith.constant 128 : i64 + %c4096_i64 = arith.constant 4096 : i64 + + %ub_in = pto.castptr %c0_i64 : i64 -> !pto.ptr + %ub_out = pto.castptr %c4096_i64 : i64 -> !pto.ptr + + pto.mte_gm_ub %input, %ub_in, %c0_i64, %c128_i64 + nburst(%c16_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64, i64 + + pto.set_flag["PIPE_MTE2", "PIPE_V", "EVENT_ID0"] + pto.wait_flag["PIPE_MTE2", "PIPE_V", "EVENT_ID0"] + + pto.vecscope { + %mask, %remaining = pto.plt_b32 %c64_i32 : i32 -> !pto.mask, i32 + + %low, %high, %next_x2 = pto.vldsx2 %ub_in[%c128], "DINTLV_B32" + : !pto.ptr, index + -> !pto.vreg<64xf32>, !pto.vreg<64xf32>, !pto.ptr + %after_x2 = pto.vlds %next_x2[%c0] + : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %after_x2, %ub_out[%c0], %mask {dist = "NORM_B32"} + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + + %block, %next_block = pto.vsldb %ub_in, %c1_i16, %c8_i16, %mask + : !pto.ptr, i16, i16, !pto.mask + -> !pto.vreg<64xf32>, !pto.ptr + %after_block = pto.vlds %next_block[%c0] + : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %after_block, %ub_out[%c64], %mask {dist = "NORM_B32"} + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + + pto.set_flag["PIPE_V", "PIPE_MTE3", "EVENT_ID0"] + pto.wait_flag["PIPE_V", "PIPE_MTE3", "EVENT_ID0"] + pto.mte_ub_gm %ub_out, %output, %c128_i64 + nburst(%c4_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64 + pto.barrier #pto.pipe + return + } +} diff --git a/test/vpto/cases/micro-op/vector-load-store/vldsx2-vsldb-post-update/launch.cpp b/test/vpto/cases/micro-op/vector-load-store/vldsx2-vsldb-post-update/launch.cpp new file mode 100644 index 0000000000..557094364c --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/vldsx2-vsldb-post-update/launch.cpp @@ -0,0 +1,25 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#ifndef __VEC_SCOPE__ +#define __VEC_SCOPE__ +#endif + +#include + +#ifndef __CPU_SIM +#include "acl/acl.h" +#endif + +extern "C" __global__[aicore] void +vldsx2_vsldb_post_update_kernel(__gm__ float *input, __gm__ float *output); + +void LaunchVldsx2VsldbPostUpdate(float *input, float *output, void *stream) { + vldsx2_vsldb_post_update_kernel<<<1, nullptr, stream>>>( + (__gm__ float *)input, (__gm__ float *)output); +} diff --git a/test/vpto/cases/micro-op/vector-load-store/vldsx2-vsldb-post-update/main.cpp b/test/vpto/cases/micro-op/vector-load-store/vldsx2-vsldb-post-update/main.cpp new file mode 100644 index 0000000000..0143388b11 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/vldsx2-vsldb-post-update/main.cpp @@ -0,0 +1,92 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#include "acl/acl.h" +#include "test_common.h" + +#include +#include + +using namespace PtoTestCommon; + +void LaunchVldsx2VsldbPostUpdate(float *input, float *output, void *stream); + +namespace { +constexpr size_t kInputElements = 512; +constexpr size_t kOutputElements = 128; +constexpr size_t kInputBytes = kInputElements * sizeof(float); +constexpr size_t kOutputBytes = kOutputElements * sizeof(float); +} // namespace + +#define ACL_CHECK(expr) \ + do { \ + const aclError ret = (expr); \ + if (ret != ACL_SUCCESS) { \ + std::fprintf(stderr, "[ERROR] %s failed: %d (%s:%d)\n", #expr, \ + static_cast(ret), __FILE__, __LINE__); \ + rc = 1; \ + goto cleanup; \ + } \ + } while (0) + +int main() { + float *inputHost = nullptr; + float *outputHost = nullptr; + float *inputDevice = nullptr; + float *outputDevice = nullptr; + aclrtStream stream = nullptr; + int rc = 0; + bool aclInited = false; + bool deviceSet = false; + int deviceId = 0; + size_t inputSize = kInputBytes; + size_t outputSize = kOutputBytes; + + ACL_CHECK(aclInit(nullptr)); + aclInited = true; + if (const char *envDevice = std::getenv("ACL_DEVICE_ID")) + deviceId = std::atoi(envDevice); + ACL_CHECK(aclrtSetDevice(deviceId)); + deviceSet = true; + ACL_CHECK(aclrtCreateStream(&stream)); + + ACL_CHECK( + aclrtMallocHost(reinterpret_cast(&inputHost), kInputBytes)); + ACL_CHECK( + aclrtMallocHost(reinterpret_cast(&outputHost), kOutputBytes)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&inputDevice), kInputBytes, + ACL_MEM_MALLOC_HUGE_FIRST)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&outputDevice), kOutputBytes, + ACL_MEM_MALLOC_HUGE_FIRST)); + + ReadFile("./input.bin", inputSize, inputHost, kInputBytes); + ReadFile("./output.bin", outputSize, outputHost, kOutputBytes); + ACL_CHECK(aclrtMemcpy(inputDevice, kInputBytes, inputHost, kInputBytes, + ACL_MEMCPY_HOST_TO_DEVICE)); + ACL_CHECK(aclrtMemcpy(outputDevice, kOutputBytes, outputHost, kOutputBytes, + ACL_MEMCPY_HOST_TO_DEVICE)); + + LaunchVldsx2VsldbPostUpdate(inputDevice, outputDevice, stream); + ACL_CHECK(aclrtSynchronizeStream(stream)); + ACL_CHECK(aclrtMemcpy(outputHost, kOutputBytes, outputDevice, kOutputBytes, + ACL_MEMCPY_DEVICE_TO_HOST)); + WriteFile("./output.bin", outputHost, kOutputBytes); + +cleanup: + aclrtFree(inputDevice); + aclrtFree(outputDevice); + aclrtFreeHost(inputHost); + aclrtFreeHost(outputHost); + if (stream != nullptr) + aclrtDestroyStream(stream); + if (deviceSet) + aclrtResetDevice(deviceId); + if (aclInited) + aclFinalize(); + return rc; +} diff --git a/test/vpto/cases/micro-op/vector-load-store/vstas-post-update/compare.py b/test/vpto/cases/micro-op/vector-load-store/vstas-post-update/compare.py new file mode 100644 index 0000000000..a9dcdd29ba --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/vstas-post-update/compare.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import numpy as np + + +def main() -> None: + golden = np.fromfile("golden_output.bin", dtype=np.float32) + output = np.fromfile("output.bin", dtype=np.float32) + if golden.shape != output.shape or not np.array_equal(golden, output): + mismatch = np.flatnonzero(golden != output) + idx = int(mismatch[0]) if mismatch.size else 0 + print( + f"[ERROR] mismatch: shape={output.shape}, idx={idx}, " + f"golden={golden[idx] if golden.size else 'n/a'}, " + f"output={output[idx] if output.size else 'n/a'}" + ) + raise SystemExit(2) + print("[INFO] compare passed") + + +if __name__ == "__main__": + main() diff --git a/test/vpto/cases/micro-op/vector-load-store/vstas-post-update/golden.py b/test/vpto/cases/micro-op/vector-load-store/vstas-post-update/golden.py new file mode 100644 index 0000000000..53e1db2afb --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/vstas-post-update/golden.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import numpy as np + + +def main() -> None: + data = (np.arange(128, dtype=np.float32) * 0.5) - 7.0 + data.tofile("input.bin") + np.zeros((64,), dtype=np.float32).tofile("output.bin") + data[64:128].tofile("golden_output.bin") + + +if __name__ == "__main__": + main() diff --git a/test/vpto/cases/micro-op/vector-load-store/vstas-post-update/kernel.pto b/test/vpto/cases/micro-op/vector-load-store/vstas-post-update/kernel.pto new file mode 100644 index 0000000000..687d6ba7d8 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/vstas-post-update/kernel.pto @@ -0,0 +1,56 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// case: micro-op/vector-load-store/vstas-post-update +// family: vector-load-store +// target_ops: pto.vstas +// scenarios: post-update-result, element-stride, empty-alignment-state + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @vstas_post_update_kernel( + %input: !pto.ptr, %output: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c64_i32 = arith.constant 64 : i32 + %c64_mask_i32 = arith.constant 64 : i32 + %c0_i64 = arith.constant 0 : i64 + %c2_i64 = arith.constant 2 : i64 + %c4_i64 = arith.constant 4 : i64 + %c128_i64 = arith.constant 128 : i64 + %c4096_i64 = arith.constant 4096 : i64 + + %ub_in = pto.castptr %c0_i64 : i64 -> !pto.ptr + %ub_out = pto.castptr %c4096_i64 : i64 -> !pto.ptr + + pto.mte_gm_ub %input, %ub_in, %c0_i64, %c128_i64 + nburst(%c4_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64, i64 + + pto.set_flag["PIPE_MTE2", "PIPE_V", "EVENT_ID0"] + pto.wait_flag["PIPE_MTE2", "PIPE_V", "EVENT_ID0"] + + pto.vecscope { + %align = pto.init_align : !pto.align + %next = pto.vstas %align, %ub_in, %c64_i32 + : !pto.align, !pto.ptr, i32 -> !pto.ptr + %mask, %remaining = pto.plt_b32 %c64_mask_i32 + : i32 -> !pto.mask, i32 + %value = pto.vlds %next[%c0] + : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %value, %ub_out[%c0], %mask {dist = "NORM_B32"} + : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + + pto.set_flag["PIPE_V", "PIPE_MTE3", "EVENT_ID0"] + pto.wait_flag["PIPE_V", "PIPE_MTE3", "EVENT_ID0"] + pto.mte_ub_gm %ub_out, %output, %c128_i64 + nburst(%c2_i64, %c128_i64, %c128_i64) + : !pto.ptr, !pto.ptr, i64, i64, i64, i64 + pto.barrier #pto.pipe + return + } +} diff --git a/test/vpto/cases/micro-op/vector-load-store/vstas-post-update/launch.cpp b/test/vpto/cases/micro-op/vector-load-store/vstas-post-update/launch.cpp new file mode 100644 index 0000000000..6d719266bf --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/vstas-post-update/launch.cpp @@ -0,0 +1,25 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#ifndef __VEC_SCOPE__ +#define __VEC_SCOPE__ +#endif + +#include + +#ifndef __CPU_SIM +#include "acl/acl.h" +#endif + +extern "C" __global__[aicore] void +vstas_post_update_kernel(__gm__ float *input, __gm__ float *output); + +void LaunchVstasPostUpdate(float *input, float *output, void *stream) { + vstas_post_update_kernel<<<1, nullptr, stream>>>((__gm__ float *)input, + (__gm__ float *)output); +} diff --git a/test/vpto/cases/micro-op/vector-load-store/vstas-post-update/main.cpp b/test/vpto/cases/micro-op/vector-load-store/vstas-post-update/main.cpp new file mode 100644 index 0000000000..ff73955a05 --- /dev/null +++ b/test/vpto/cases/micro-op/vector-load-store/vstas-post-update/main.cpp @@ -0,0 +1,92 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#include "acl/acl.h" +#include "test_common.h" + +#include +#include + +using namespace PtoTestCommon; + +void LaunchVstasPostUpdate(float *input, float *output, void *stream); + +namespace { +constexpr size_t kInputElements = 128; +constexpr size_t kOutputElements = 64; +constexpr size_t kInputBytes = kInputElements * sizeof(float); +constexpr size_t kOutputBytes = kOutputElements * sizeof(float); +} // namespace + +#define ACL_CHECK(expr) \ + do { \ + const aclError ret = (expr); \ + if (ret != ACL_SUCCESS) { \ + std::fprintf(stderr, "[ERROR] %s failed: %d (%s:%d)\n", #expr, \ + static_cast(ret), __FILE__, __LINE__); \ + rc = 1; \ + goto cleanup; \ + } \ + } while (0) + +int main() { + float *inputHost = nullptr; + float *outputHost = nullptr; + float *inputDevice = nullptr; + float *outputDevice = nullptr; + aclrtStream stream = nullptr; + int rc = 0; + bool aclInited = false; + bool deviceSet = false; + int deviceId = 0; + size_t inputSize = kInputBytes; + size_t outputSize = kOutputBytes; + + ACL_CHECK(aclInit(nullptr)); + aclInited = true; + if (const char *envDevice = std::getenv("ACL_DEVICE_ID")) + deviceId = std::atoi(envDevice); + ACL_CHECK(aclrtSetDevice(deviceId)); + deviceSet = true; + ACL_CHECK(aclrtCreateStream(&stream)); + + ACL_CHECK( + aclrtMallocHost(reinterpret_cast(&inputHost), kInputBytes)); + ACL_CHECK( + aclrtMallocHost(reinterpret_cast(&outputHost), kOutputBytes)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&inputDevice), kInputBytes, + ACL_MEM_MALLOC_HUGE_FIRST)); + ACL_CHECK(aclrtMalloc(reinterpret_cast(&outputDevice), kOutputBytes, + ACL_MEM_MALLOC_HUGE_FIRST)); + + ReadFile("./input.bin", inputSize, inputHost, kInputBytes); + ReadFile("./output.bin", outputSize, outputHost, kOutputBytes); + ACL_CHECK(aclrtMemcpy(inputDevice, kInputBytes, inputHost, kInputBytes, + ACL_MEMCPY_HOST_TO_DEVICE)); + ACL_CHECK(aclrtMemcpy(outputDevice, kOutputBytes, outputHost, kOutputBytes, + ACL_MEMCPY_HOST_TO_DEVICE)); + + LaunchVstasPostUpdate(inputDevice, outputDevice, stream); + ACL_CHECK(aclrtSynchronizeStream(stream)); + ACL_CHECK(aclrtMemcpy(outputHost, kOutputBytes, outputDevice, kOutputBytes, + ACL_MEMCPY_DEVICE_TO_HOST)); + WriteFile("./output.bin", outputHost, kOutputBytes); + +cleanup: + aclrtFree(inputDevice); + aclrtFree(outputDevice); + aclrtFreeHost(inputHost); + aclrtFreeHost(outputHost); + if (stream != nullptr) + aclrtDestroyStream(stream); + if (deviceSet) + aclrtResetDevice(deviceId); + if (aclInited) + aclFinalize(); + return rc; +}