From 716f624907052e4d83149590a82ed0e9a80e2846 Mon Sep 17 00:00:00 2001 From: yexiaosu Date: Tue, 28 Jul 2026 09:28:33 +0800 Subject: [PATCH 1/3] feat(vpto): add sequential soft post-update analysis --- .../designs/vpto-soft-postupdate-design-zh.md | 218 +++-- include/PTO/Transforms/Passes.td | 9 +- lib/PTO/Transforms/VPTOSoftPostUpdate.cpp | 761 +++++++++++++++--- .../soft_postupdate_sequential-base-chain.pto | 51 ++ ...postupdate_sequential-bucket-isolation.pto | 71 ++ ...ft_postupdate_sequential-nested-blocks.pto | 67 ++ ...soft_postupdate_sequential-run-restart.pto | 70 ++ ...ft_postupdate_sequential-symbolic-leaf.pto | 85 ++ ...ft_postupdate_sequential-vsstb-initial.pto | 60 ++ .../vpto/soft_postupdate_sequential-vsstb.pto | 65 ++ test/lit/vpto/soft_postupdate_sequential.pto | 217 +++++ 11 files changed, 1502 insertions(+), 172 deletions(-) create mode 100644 test/lit/vpto/soft_postupdate_sequential-base-chain.pto create mode 100644 test/lit/vpto/soft_postupdate_sequential-bucket-isolation.pto create mode 100644 test/lit/vpto/soft_postupdate_sequential-nested-blocks.pto create mode 100644 test/lit/vpto/soft_postupdate_sequential-run-restart.pto create mode 100644 test/lit/vpto/soft_postupdate_sequential-symbolic-leaf.pto create mode 100644 test/lit/vpto/soft_postupdate_sequential-vsstb-initial.pto create mode 100644 test/lit/vpto/soft_postupdate_sequential-vsstb.pto create mode 100644 test/lit/vpto/soft_postupdate_sequential.pto diff --git a/docs/designs/vpto-soft-postupdate-design-zh.md b/docs/designs/vpto-soft-postupdate-design-zh.md index d53d67ad7..9a1542d52 100644 --- a/docs/designs/vpto-soft-postupdate-design-zh.md +++ b/docs/designs/vpto-soft-postupdate-design-zh.md @@ -2,7 +2,7 @@ ## 1. 概述 -本文档设计一个新的 PTOAS pass——`VPTOSoftPostUpdate`,在 **MLIR 层**(LLVM lowering 之前)将非 Post-Update 形式的 VPTO 访存操作转换为 Post-Update 形式。首期目标指令为已支持 `updated_base` 的三个 op(`pto.vlds`、`pto.vsts`、`pto.vsstb`),后续扩展至更多访存指令(见第 2 节指令全景)。pass 覆盖两种场景:`scf.for` 循环内的固定步长访存模式(循环路径)和前端已展开的连续等差访存序列(顺序路径)。 +本文档设计一个新的 PTOAS pass——`VPTOSoftPostUpdate`,在 **MLIR 层**(LLVM lowering 之前)将非 Post-Update 形式的 VPTO 访存操作转换为 Post-Update 形式。首期目标指令为已支持 `updated_base` 的三个 op(`pto.vlds`、`pto.vsts`、`pto.vsstb`),后续扩展至更多访存指令(见第 2 节指令全景)。pass 覆盖两种场景:`scf.for` 循环间的固定步长访存模式(循环路径),以及同一 block 单次执行中的等步长访存 run(顺序路径)。 循环路径示例: @@ -106,33 +106,33 @@ bisheng 的 pass 工作在 LLVM IR 上,此时高级循环结构和 PTO 类型 | `pto.vsstb` 块步长存储 | 部分支持(独立开关) | ✓(统一框架) | | AIV 软件循环 | ✗(排除,因为 AIV 软件循环不被 `LoopInfo` 识别为真实硬件循环,SCEV 无法分析) | ✓(`scf.for` 统一表示,无此限制) | | 非循环顺序访问 | ✗(不支持) | ✓(顺序路径,见 4.3) | +| 循环体内单次迭代的展开访问 | ✗(不支持) | ✓(循环路径未命中的剩余 op 由顺序路径处理) | | 显式 `arith.addi` 步长模式 | 可能 SCEV 失败 | ✓(直接匹配 `arith.addi`) | ## 4. 算法设计 ### 4.1 整体流程 +pass 的驱动分为两个阶段。两个阶段都通过 `PostUpdateTable`(static `StringMap`,维护第 2 章的指令集合及其地址操作数布局)识别候选,并共享地址单位换算、`StrideExpr`、合法性检查和 post-update op 构造逻辑: + ``` -VPTOSoftPostUpdate pass: - 单次遍历 pto.vecscope 内所有指令,对每条指令查询 - PostUpdateSet(static DenseSet,维护第 2 章中的指令集合): - 若命中: - 若在 scf.for 内 → 循环路径(分析与物化两阶段): - 阶段一 · 纯符号分析(不产生任何 IR): - 1. 累加器分析 —— offset 或 base 是否为 iter_arg 且 yield 中有显式递增? - 2. 若累加器分析未命中 → delta 分析 —— stride 是否为循环不变量? - 阶段二 · 判定与物化: - 3. 合法性检查(含类型一致性、操作数可用性) - 4. 在唯一插入点物化 stride,然后改写 - 5. 若产生了新的 iter_arg,向外层循环传播 - 若不在 scf.for 内 → 顺序路径: - 从当前指令起向后扫描,收集连续的同类型、同 base、等差常量偏移的序列, - 一次性链式改写为 post-update,迭代器跳过已处理的指令 +阶段一 · 循环路径:候选 op 直接位于某个 scf.for 的循环体内,并且在每次迭代中更新。 + 以 ForOp 为单位批量处理(同循环内多 op 合并 iter_arg): + · 纯符号分析:先做累加器分析,未命中再做 delta 分析。 + · 判定与物化:完成合法性检查后,在唯一插入点物化 stride 并改写; + 若产生新的 iter_arg,则向外层循环传播。 + +阶段二 · 顺序路径:循环路径完成后,重新收集 vecscope 内的全部 block, + 包括 scf.for body、vecscope 体和 scf.if 分支体。只扫描仍为非 Post-Update + 形式的剩余 op;以 block 为单位分桶,桶内按程序序寻找互不重叠的 + maximal arithmetic runs,并链式改写。 ``` +循环路径按内层到外层处理全部 `scf.for`。由于循环改写会 erase 并重建 ForOp,顺序路径必须在全部循环改写结束后重新收集 block 和候选 op,不能跨阶段保存 `Block *` 或 `Operation *`。这样循环路径已改写的 op 会因已有 `updated_base` 被自然跳过;循环路径未命中,以及原本位于 `scf.if` 等嵌套区域中的 op,仍可在单次 block 执行范围内参与顺序路径。 + ### 4.2 循环路径 -循环路径对 base 和 strideOperand **各自独立**分析:每个操作数先试 **累加器分析**(优先),未命中再退到 **delta 分析**(兜底),两者的结果最后合并为 `total_delta`。前者处理该操作数已通过 `iter_args` 显式累加的场景(stride 可以是任意已计算的值),后者处理从 IV 全新计算、无累加器的场景(stride 须为循环不变量)。因此同一条指令的 base 走累加器、strideOperand 走 delta 是允许的组合。 +循环路径对 base 和 strideOperand **各自独立**分析:每个操作数先试 **累加器分析**(优先),未命中再退到 **delta 分析**(兜底),两者的结果最后按 4.2.1 的公式合并为 `stride_new`。前者处理该操作数已通过 `iter_args` 显式累加的场景(stride 可以是任意已计算的值),后者处理从 IV 全新计算、无累加器的场景(stride 须为循环不变量)。因此同一条指令的 base 走累加器、strideOperand 走 delta 是允许的组合。 两类指令(vlds/vsts 与 vsstb/vsldb)的分析和改写通过统一的地址描述符抽象,共享同一套分析流程。 @@ -140,30 +140,42 @@ VPTOSoftPostUpdate pass: #### 4.2.1 地址描述符 -每个候选 op 的有效地址可表示为 `base + weight * strideOperand`: +每个候选 op 由 `PostUpdateOpInfo` 描述:base 与 strideOperand 的操作数下标,以及 strideOperand 的**单位**。 ``` -struct OpAddressDescriptor { - Value base; // dest / source - Value strideOperand; // offset(vlds/vsts)/ repeat_stride(vsstb/vsldb) - int64_t weight; // 1(vlds/vsts)/ 32(vsstb/vsldb) +enum class StrideUnit { Element, Block, Byte }; +struct PostUpdateOpInfo { + int baseOperandIdx; + int strideOperandIdx; + StrideUnit strideUnit; + unsigned minResultsForPost; }; ``` -| 指令 | base | strideOperand | weight | 有效地址 | -|------|------|---------------|--------|---------| -| vlds/vsts | source/destination | offset (Index) | 1 | base + offset | -| vsstb/vsldb | destination/source | repeat_stride (I16) | 32 | dest + 32*repeat_stride | +`delta(base)` 与 `strideOperand` 的单位不同,合并前必须先统一到字节。引入两个字节量: + +- **`elemBytes`** = base 指针一个 `pto.addptr` 单位的字节数(由 `addPtrUnitBytes(base)` 求得,即 lowering 规约后的 GEP 元素类型宽度;非字节对齐的低精度打包类型无法确定,直接放弃候选)。 +- **`unitBytes`** = strideOperand 一个单位的字节数(由 `strideUnitBytes(unit, elemBytes)` 求得)。单位取值来自该 op 的 lowering: + +| 指令 | base | strideOperand | strideUnit | unitBytes | 有效地址 | +|------|------|---------------|-----------|-----------|---------| +| vlds/vsts | source/destination | offset (Index) | Element | elemBytes | base + offset | +| vsstb/vsldb | destination/source | repeat_stride (I16) | Block | 32 | dest + (32/elemBytes)·repeat_stride | +| sprsts/sprsti(Step 4) | destination | offset (I32) | Byte | 1 | dest + offset/elemBytes | + +> 新增指令时判断单位的方法:看它在 `VPTOLLVMEmitter.cpp` 的 lowering pattern 里,strideOperand 是否过 `convertElementOffsetToBytes`(→ Element),是否经 `packBlockRepeatStride` 透传控制字(→ Block),还是原样透传且 ISA 文档标注为字节(→ Byte)。 -分析和改写的核心公式基于此描述符统一表达: +分析和改写的核心公式统一以字节表达: ``` -total_delta = delta(base) + weight * delta(strideOperand) -stride_new = total_delta / weight // 填入 post-update 的 strideOperand 槽 -init_ptr = base_0 + weight * strideOperand_0 // 通过 pto.addptr 创建 +Δbytes = elemBytes·delta(base) + unitBytes·delta(strideOperand) +stride_new = Δbytes / unitBytes = (elemBytes/unitBytes)·delta(base) + delta(strideOperand) +init_ptr = pto.addptr(base_0, (unitBytes/elemBytes)·strideOperand_0) // 偏移以元素计 ``` -约束:`total_delta` 必须整除 `weight`(weight=1 时自动满足,weight=32 时要求 `delta(base) % 32 == 0`,因为 `weight * delta(strideOperand)` 天然整除 32)。 +展开后可见**只有 `delta(base)` 被缩放,`delta(strideOperand)` 恒等透传**——这保证了 stride 保持符号形式,循环变化的增量对所有指令类别都仍受支持;当 `elemBytes == unitBytes`(Element 类)缩放因子为 1,不发射任何 IR,vlds/vsts 的行为与未引入缩放前逐字相同。 + +约束(由 4.2.5 检查 3 统一裁定):`(elemBytes/unitBytes)·delta(base)` 须为精确的整数缩放。`unitBytes % elemBytes == 0` 时,将 `delta(base)` 规范化为“常量 + Σ(系数 × SSA 叶子)”,要求常量和每个系数都能被 `unitBytes/elemBytes` 整除;`elemBytes % unitBytes == 0` 时乘 `elemBytes/unitBytes` 恒精确;互不整除则放弃。`init_ptr` 的 `(unitBytes/elemBytes)·strideOperand_0` 对称:Block 类恒精确(elemBytes 整除 32),Byte 类要求 `elemBytes | strideOperand_0`。 #### 4.2.2 分析与物化分离 @@ -176,7 +188,7 @@ StrideExpr := Const(int64) // 类型在物化时按上下文决 | Cast(op, e) // index_cast/index_castui,op 为克隆模板 ``` -`StrideExpr` 在构造时即做常量折叠(`foldConst`),因此 `total_delta` 是否为常量、是否整除 `weight`、是否恒为零,全部在符号层判定,判定不依赖任何已生成的 IR。 +`StrideExpr` 在构造时即做常量折叠(`foldConst`),因此 `delta(base)` 缩放是否精确、`stride_new` 是否恒为零,全部在符号层判定,判定不依赖任何已生成的 IR。 该划分是正确性要求,而非代码组织风格。以下三条性质依赖于它: @@ -186,7 +198,7 @@ StrideExpr := Const(int64) // 类型在物化时按上下文决 2. **递归可 memoize。** 纯函数的结果只取决于入参,`decomposeLinear` 与 `computeDelta` 按 `Value` 缓存,共享子表达式只分解一次,分解代价与定义链 DAG 的规模成线性关系。 -3. **被拒候选不留残留。** 合法性检查全部先于物化完成,放弃某个候选时 IR 尚未被触碰。`weight` 缩放这类中间运算也只存在于符号层,不会以 op 的形式落地。 +3. **被拒候选不留残留。** 合法性检查全部先于物化完成,放弃某个候选时 IR 尚未被触碰。`elemBytes/unitBytes` 缩放这类中间运算也只存在于符号层,不会以 op 的形式落地。 #### 4.2.3 累加器分析(优先) @@ -222,13 +234,12 @@ baseIncr = getIterArgIncrement(desc.base, forOp) // NotIterArg → 回退 d soIncr = getIterArgIncrement(desc.strideOperand, forOp) 任一为 Failed → 放弃该候选 -否则 → total_delta = delta(base) + weight * delta(strideOperand) // 符号相加 - stride_new = total_delta / weight +否则 → stride_new = (elemBytes/unitBytes)·delta(base) + delta(strideOperand) // 见 4.2.1,符号相加 ``` 不要求 stride 是常量或循环不变量;增量也不要求定义在候选 op 之前——不满足时由 4.2.5 的可用性检查决定克隆或放弃。 -**示例(vlds,weight=1):** +**示例(vlds,Element 类,elemBytes == unitBytes):** ```mlir %base = pto.castptr %c0_i64 : i64 -> !pto.ptr @@ -240,7 +251,7 @@ scf.for %iv = %c0 to %c16 step %c1 scf.yield %next_off } ``` -`baseIncr` = `NotIterArg`(回退 delta,得 `Const(0)`),`soIncr` = `{Ok, Leaf(%s)}`。`total_delta = Leaf(%s)`,`stride_new = %s / 1 = %s`。 +`baseIncr` = `NotIterArg`(回退 delta,得 `Const(0)`),`soIncr` = `{Ok, Leaf(%s)}`。elemBytes == unitBytes,`stride_new = (elemBytes/unitBytes)·0 + %s = %s`。 注意 `%s` 在本例中定义于 `pto.vlds` 之前,但这不是前提:若 `%s` 定义在 `pto.vlds` 之后,4.2.5 会把它的定义链克隆到候选 op 之前,结果不变。 @@ -261,11 +272,10 @@ scf.for %iv = %c0 to %c16 step %c1 | 其他 | `unknown`(放弃) | ``` -total_delta = delta(base) + weight * delta(strideOperand) -stride_new = total_delta / weight +stride_new = (elemBytes/unitBytes)·delta(base) + delta(strideOperand) // 见 4.2.1 ``` -`stride_new` 须为循环不变量,`total_delta` 须整除 `weight`。 +`stride_new` 须为循环不变量,`(elemBytes/unitBytes)·delta(base)` 须为精确整数缩放(见 4.2.1 约束)。 **正确性:** delta 表中的操作构成仿射函数的封闭运算集合。定义链仅由这些操作构成时,delta 计算不会遗漏。遇到表外操作时保守放弃。 @@ -279,9 +289,9 @@ delta 分析同样是纯符号的:表中每一行返回 `StrideExpr`,结果 2. **op 直接位于 `scf.for` 循环体内**(不嵌套在循环内的 `scf.if` 或其他控制流中),避免部分迭代问题。 -3. **`total_delta` 整除 `weight`。** weight=1 时自动满足。weight≠1 时,**当前实现要求整个 `total_delta` 是编译期常量**且整除 weight,不满足即放弃。 +3. **`(elemBytes/unitBytes)·delta(base)` 为精确整数缩放。** Element 类(elemBytes == unitBytes)自动满足;Block 类(`unitBytes % elemBytes == 0`,如 unitBytes=32)要求规范化 `delta(base)` 的常量项和每个 SSA 叶子系数都能被 `unitBytes/elemBytes` 整除;Byte 类(`elemBytes % unitBytes == 0`)乘 `elemBytes/unitBytes` 恒精确;互不整除则放弃。 - 这比理论下限保守。由于 `weight * delta(strideOperand)` 天然整除 weight,整除性其实只取决于 `delta(base)`——原则上 `delta(strideOperand)` 可以是任意符号表达式,只要 `delta(base)` 是能被 weight 整除的常量即可。放宽需要给 `StrideExpr` 增加结构化的整除判定(识别 `Mul(e, Const(k))` 一类形式),暂未实现;此处记录该取舍,以免文档与实现口径不一致。 + 关键在于**只有 `delta(base)` 参与缩放**:`delta(strideOperand)` 恒等透传、不要求整除也不要求是常量,因此循环变化的 strideOperand 增量对各类指令都受支持(如 vsstb 的 `repeat_stride` 为非常量 iter_arg 累加)。仿射规范化使 `8*k` 这类非常量 base delta 也能证明可被 8 精确缩放。 4. **stride_new 为零时跳过。** 地址不前进,post-update 无收益。常量折叠使各项相消、合成结果恒为零的情形(如 base 每轮 +8、strideOperand 每轮 −8)同样能被识别。 @@ -297,7 +307,7 @@ delta 分析同样是纯符号的:表中每一行返回 `StrideExpr`,结果 1. **物化 stride。** 叶子全部循环不变时发射到循环外,否则发射到候选 op 之前。常量按 `(值, 类型)` 在同一循环内复用同一个 SSA 值——4.2.7 的分组按 `(base, stride_new)` 的 **Value 同一性** 判定,重复创建等值常量会把本可共享 `iter_arg` 的 op 拆成多组。 -2. 计算初始指针 `init_ptr = base_0 + weight * strideOperand_0`(通过 `pto.addptr`;若值为零则直接用 `base_0`)。 +2. 计算初始指针 `init_ptr = pto.addptr(base_0, (unitBytes/elemBytes)·strideOperand_0)`(见 4.2.1;若偏移为零则直接用 `base_0`)。 3. 新增指针类型的 `iter_arg`,初始值为 `init_ptr`。 4. 创建 Post-Update op:将 `strideOperand` 替换为 `stride_new`,base 替换为 iter_arg 的 block argument。其余操作数(block_stride、mask、dist 等)不变。 5. 将 `updated_base` 通过 `scf.yield` 传出。 @@ -308,11 +318,11 @@ delta 分析同样是纯符号的:表中每一行返回 `StrideExpr`,结果 非 Post-Update:`dest + 32*repeat_stride + blk*32*block_stride` Post-Update:`dest_p + blk*32*block_stride`(repeat_stride 不参与存储地址),返回 `dest_p + 32*repeat_stride_p` -Post-Update 模式下 `repeat_stride` 从地址偏移变为指针前进量,因此初始指针需吸收原始偏移 `32*rs_0`。 +Post-Update 模式下 `repeat_stride` 从地址偏移变为指针前进量,因此初始指针需吸收原始偏移 `32*rs_0` 字节,即 `pto.addptr` 偏移 `(32/elemBytes)*rs_0` 个元素(见 4.2.1)。 #### 4.2.7 同一循环中的多个 Op -两个 op 能共享同一个 `iter_arg`,当且仅当它们走**同一条地址序列**——起点 `init_ptr = base_0 + weight * strideOperand_0` 相同,且步长 `stride_new` 相同。 +两个 op 能共享同一个 `iter_arg`,当且仅当它们走**同一条地址序列**——起点 `init_ptr`(由 `base_0` 与 `strideOperand_0` 决定,见 4.2.1)相同,且步长 `stride_new` 相同。 理想的分组键是 `(init_ptr, stride_new)`。但 `init_ptr` 不适合直接入键:分组按 **Value 同一性** 比较,而 `computeInitialPtr` 可能为每个候选各自物化一个 `pto.addptr`,起点数值相同也未必是同一个 SSA 值。因此改用决定 `init_ptr` 的**原始操作数**:分组键取 `(base, strideOperand, stride_new)`。操作数相同必然起点相同,这是一个充分条件——可能把本可合并的组拆开,但绝不会合并本应分开的组。 @@ -326,37 +336,89 @@ Post-Update 模式下 `repeat_stride` 从地址偏移变为指针前进量,因 ### 4.3 顺序路径 -处理前端已展开循环(或非循环场景)中连续排列的同类访存指令。 +顺序路径处理同一 block 单次执行中的等步长访问,包括非循环代码,以及 `scf.for` body 中循环路径未命中的剩余 op。步长只需在本次 block 执行期间保持不变,可以是编译期常量,也可以是由 block argument、循环 IV、iter_arg 或其他已存在 SSA 值组成的仿射表达式。 + +顺序路径复用循环路径的 `StrideExpr`、`scaleBaseDelta` / `combineStride`、类型与可用性检查、pure 定义链克隆、初始指针计算和 post-update op 构造。新增逻辑仅包括 base 归一化、仿射表达式规范化以及 block 内 run 检测。 + +#### 4.3.1 有效地址分析 + +为比较 base 不同的候选,先沿 `pto.addptr` 链把每个 base 归一为: -#### 4.3.1 序列检测 +``` +base_i = pto.addptr(rootBase, baseOffset_i) // baseOffset_i 以元素计 +``` + +若 base 不是 `pto.addptr` 的结果,则 `rootBase = base`、`baseOffset = 0`。`baseOffset` 和 strideOperand 沿与循环路径相同的 `arith.addi`、`arith.subi`、常量乘法和 index cast 规则构造成 `StrideExpr`;无法继续分解的 SSA 值作为叶子保留。 + +对同一 op 类型、同一 `rootBase` 的相邻候选 `i-1` 和 `i`,定义: + +``` +deltaBase = baseOffset_i - baseOffset_(i-1) // 元素 +deltaStride = strideOperand_i - strideOperand_(i-1) // strideOperand 单位 + +step = combineStride(deltaBase, deltaStride, elemBytes, unitBytes) + = (elemBytes/unitBytes)·deltaBase + deltaStride +``` -同样使用地址描述符。遍历过程中命中一条不在 `scf.for` 内的候选 op 时,提取其描述符,然后向后扫描同一 block 内的后续指令,收集满足以下条件的最长序列: +`step` 始终以该 op 的 strideOperand 单位表示。单位换算和精确缩放直接复用 4.2.1 与 4.2.5 的规则,不能证明精确换算时放弃该相邻关系。 -- 同一 op 类型 -- 同一 base 操作数 -- 相邻 op 的有效地址 `base_i + weight * strideOperand_i` 构成等差数列 -- 等差公差整除 `weight` +为比较两个非常量 `step` 是否相同,将 `StrideExpr` 按加减、常量乘法进行展平和常量折叠,得到规范化仿射形式“常量 + Σ(系数 × SSA 叶子)”。相同叶子按 Value 同一性合并,系数为零的项删除。两个 step 的规范化形式完全相同,才视为固定公差;不支持的非仿射运算保守地作为叶子,仅在复用同一 SSA 值时才能匹配。 + +该分析可识别 base 与 offset 相互补偿的模式。例如 Element 类指令的三个候选满足: + +``` +baseOffset: x, x + 16, x + 32 +offset: -x, 48 - x, 96 - x +有效偏移: 0, 64, 128 +``` + +虽然 base 和 offset 都不是常量,但相邻地址的规范化 `step` 恒为 64。 + +#### 4.3.2 分桶与 maximal runs + +在一个 block 内,将仍未改写的候选按 `(op 类型, rootBase)` 放入有序桶。不同桶的 op 可以在物理程序序中任意交错;每个桶只保留本桶候选的原始程序序。 + +每个桶使用一次确定性的线性扫描,产生互不重叠的 maximal arithmetic runs: + +1. 以当前候选作为 run 起点,下一候选与它形成第一个非零、合法的 `step`。 +2. 后续候选与前一候选的 `step` 若和当前 run 的规范化 step 相同,则加入 run。 +3. 若 step 不同或无法分析,则当前 run 立即结束;破坏公差的候选作为下一个 run 的起点。 +4. 尚未形成合法 step 的两个候选若无法配对,丢弃前一个,以后一个重新尝试。 +5. run 长度达到 2 时即可确定 `step`,但长度至少为 3 才改写。已经归入已完成 run 的候选不再参与后续 run,因此不存在重叠、回溯或最长子序列冲突。 + +该规则不追求全局最多命中,而是保证结果简单、线性、确定且符合程序序。 ```mlir -// vlds 示例(weight=1):offset 构成等差 -%v0 = pto.vlds %base[%c0] : ... // 有效地址 = base + 0 -%v1 = pto.vlds %base[%c64] : ... // 有效地址 = base + 64 -%v2 = pto.vlds %base[%c128] : ... // 有效地址 = base + 128 -// 公差 = 64,stride_new = 64 / 1 = 64 +// vlds 桶:中间可以穿插其他桶的 op +%v0 = pto.vlds %base0[%off0] : ... // 本桶地址 0 +pto.vsts %x, %other[%c0], %mask : ... +%v1 = pto.vlds %base1[%off1] : ... // 本桶地址 64 +%v2 = pto.vlds %base2[%off2] : ... // 本桶地址 128 +// 本桶形成一个 step = 64 的 maximal run ``` -序列长度至少为 2 才值得变换。 +> **与循环路径的关键区别(链式 vs 共享)。** 循环路径同组 op 共享一个 `iter_arg`、全用 pre-update 指针、**不**链式传递(4.2.7,同迭代内同地址)。顺序路径相反,**必须**链式传递(前一条的 `updated_base` 喂后一条,见 4.3.4)。因此两条交错的序列(vlds 链 + vsts 链)是两条**独立**的链,各自穿针,不能合并;按 `(op 类型, rootBase)` 分桶使每条链自然独立。 + +#### 4.3.3 合法性检查 + +1. **op 尚未处于 Post-Update 形式。** 循环路径已经改写的 op 自动排除。 +2. **地址可归一到同一 rootBase。** 只穿过 `pto.addptr`;其他指针变换不猜测别名关系。 +3. **step 可精确换算且非零。** 复用 4.2.5 的单位、常量折叠和目标类型可表示性检查。 +4. **step 在 run 头可用。** 复用 4.2.5 的叶子可用性检查;定义在 run 头之后的 pure 仿射定义链可克隆到 run 头之前,含副作用或无法支配的定义使该 run 放弃。 +5. **整条 run 先分析、后物化。** 所有候选通过检查后才创建常量、`pto.addptr` 或新 op,拒绝 run 不留下残余 IR。 -#### 4.3.2 合法性检查 +顺序路径不重排 op,也不改变访问地址和访问顺序,因此不同候选之间的内存读写不会截断 run,无需额外别名分析。 -1. **op 尚未处于 Post-Update 形式。** -2. **公差整除 `weight`。** -3. **stride_new 为编译期常量。** -4. **序列中相邻指令之间无对同一 base 的别名写入。** 若序列中间插入了对 base 指向内存的写操作,序列在该处截断。 +#### 4.3.4 改写 -#### 4.3.3 改写 +对每个 run,`stride_new` 就是 4.3.1 得到的规范化 `step`。初始指针直接从 run 首条 op 的实际操作数计算: -计算 `init_ptr = base_0 + weight * strideOperand_0`(首条 op 的有效地址),`stride_new = 公差 / weight`。将序列中的每条 op 替换为 post-update 形式,前一条的 `updated_base` 作为后一条的 base,strideOperand 替换为 `stride_new`: +``` +init_ptr = pto.addptr(first.base, + (unitBytes/elemBytes)·first.strideOperand) +``` + +这里复用循环路径的精确单位换算;不需要从 `rootBase` 重新构造绝对地址。将 run 中每条 op 替换为 post-update 形式,前一条的 `updated_base` 作为后一条的 base,strideOperand 替换为 `stride_new`: ```mlir // vlds 变换后 @@ -365,7 +427,7 @@ Post-Update 模式下 `repeat_stride` 从地址偏移变为指针前进量,因 %v2, %ptr3 = pto.vlds %ptr2[%c64] : ... -> ..., !pto.ptr ``` -改写完成后,迭代器跳过已处理的指令。 +所有 run 在只读分析阶段确定后,先物化各自的 `stride_new` 和 `init_ptr`,再按原程序序替换候选 op,避免 erase op 使其他 run 保存的表达式失效。不同桶各自维护独立指针链,即使其 op 在原 block 中交错也互不影响。 ## 5. Pass 集成 @@ -378,8 +440,6 @@ VPTOExpandWrapperOps CSE PTOInferVPTOVecScope → VPTOSoftPostUpdate(新增) ← 在此处 -Canonicalizer -CSE ... PrepareVPTOLLVMLoweringPass LowerVPTOOpsPass @@ -428,18 +488,22 @@ def VPTOSoftPostUpdate : Pass<"vpto-soft-postupdate", "ModuleOp"> { ### Step 3:顺序路径 -16. 实现顺序路径的序列检测(4.3.1)和合法性检查(4.3.2)。 -17. 实现链式改写(4.3.3)和迭代器跳过逻辑。 -18. 添加顺序路径的 lit 测试。 +16. 抽取循环与顺序路径共享的 helper:精确 elemBytes/unitBytes 换算、`combineStride`、初始指针计算和 post-update op 泛型构造。 +17. 实现 base 归一化:沿 `pto.addptr` 得到 `(rootBase, baseOffset)`;复用 `StrideExpr` 构造规则,并增加仿射规范化、相等比较和按系数精确缩放,使两条路径都支持 `8*k` 一类可证明整除的非常量表达式。 +18. 调整驱动为两阶段:先内到外完成全部循环路径,再重新收集所有 block(包括 `scf.for` body)和剩余非 Post-Update op。 +19. 实现有序分桶与线性 maximal-run 检测:按 `(op 类型, rootBase)` 分桶,允许不同桶物理交错,run 互不重叠且不回溯。 +20. 复用循环路径的类型、精确换算、可用性和 pure 克隆检查,在整条 run 通过后物化 step 并链式改写。 +21. 更新 `Passes.td` 中 pass 描述,使其同时覆盖循环递推和 block 内顺序访问。 +22. 添加 `test/lit/vpto` 回归测试:固定 base 常量序列、变化 base 与 offset 抵消、非常量仿射 step、Block 类 `8*k` 精确缩放、for-body 循环路径未命中后转顺序路径、不同桶交错、多 maximal runs、公差破坏、零步长及无法精确换算的负向用例。 ### Step 4:扩展指令覆盖 -19. 为 2.2 中的指令(`vldsx2`、`vsldb`、`plds`、`pldi` 等)添加 ODS `updated_base` 定义。 -20. 扩展 PostUpdateSet,使 pass 覆盖新指令。 -21. 补充对应的 LLVM lowering(post intrinsic callee)和 lit 测试。 +23. 为 2.2 中的指令(`vldsx2`、`vsldb`、`plds`、`pldi` 等)添加 ODS `updated_base` 定义。 +24. 扩展 `PostUpdateTable`,为每条新指令按 4.2.1 的方法确定 `StrideUnit`(Element / Block / Byte)。 +25. 补充对应的 LLVM lowering(post intrinsic callee)和 lit 测试。 ### Step 5:验证与开启 -22. 端到端验证:用 ptoas 编译,与 bisheng Post-Update 输出对比已知 kernel。 -23. NPU 验证:在硬件上运行 Post-Update kernel(通过现有 `test/vpto/cases/micro-op/vector-load-store/` 框架)。 -24. 将默认值切换为开启。 +26. 端到端验证:用 ptoas 编译,与 bisheng Post-Update 输出对比已知 kernel。 +27. NPU 验证:在硬件上运行 Post-Update kernel(通过现有 `test/vpto/cases/micro-op/vector-load-store/` 框架)。 +28. 将默认值切换为开启。 diff --git a/include/PTO/Transforms/Passes.td b/include/PTO/Transforms/Passes.td index 22bacaabe..f4b965dc5 100644 --- a/include/PTO/Transforms/Passes.td +++ b/include/PTO/Transforms/Passes.td @@ -1180,11 +1180,10 @@ def VPTOSoftPostUpdate : Pass<"vpto-soft-postupdate", "ModuleOp"> { let summary = "Convert fixed-stride VPTO memory ops to post-update form"; let description = [{ - Identifies VPTO load/store operations (vlds, vsts, vsstb) inside scf.for - loops whose effective address advances by a loop-invariant stride per - iteration, and rewrites them to use the post-update (updated_base) result. - The base pointer is carried through scf.for iter_args, eliminating explicit - address arithmetic in the loop body. + Identifies VPTO load/store operations (vlds, vsts, vsstb) whose effective + address advances by a fixed symbolic stride. Loop-carried accesses use + scf.for iter_args; remaining arithmetic runs within each block, including + loop bodies, are rewritten as updated_base pointer chains. }]; let constructor = "mlir::pto::createVPTOSoftPostUpdatePass()"; let dependentDialects = ["mlir::func::FuncDialect", diff --git a/lib/PTO/Transforms/VPTOSoftPostUpdate.cpp b/lib/PTO/Transforms/VPTOSoftPostUpdate.cpp index d82c78b1d..1095b3fdd 100644 --- a/lib/PTO/Transforms/VPTOSoftPostUpdate.cpp +++ b/lib/PTO/Transforms/VPTOSoftPostUpdate.cpp @@ -12,6 +12,7 @@ #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/SCF/IR/SCF.h" #include "mlir/IR/BuiltinOps.h" +#include "mlir/IR/Dominance.h" #include "mlir/IR/IRMapping.h" #include "mlir/IR/PatternMatch.h" #include "mlir/Interfaces/SideEffectInterfaces.h" @@ -46,13 +47,13 @@ enum class StrideUnit { Byte, // sprsts/sprsti: raw byte offset }; -// Per-op-type descriptor: how to extract address operands and check post-update. -// base/strideOperand indices are operand positions. +// Per-op-type descriptor: how to extract address operands and check +// post-update. base/strideOperand indices are operand positions. struct PostUpdateOpInfo { int baseOperandIdx; int strideOperandIdx; StrideUnit strideUnit; - unsigned minResultsForPost; // numResults > this means already post-update + unsigned minResultsForPost; // numResults > this means already post-update }; using PostUpdateTable = llvm::StringMap; @@ -61,9 +62,9 @@ static const PostUpdateTable &getPostUpdateTable() { static const PostUpdateTable table = [] { PostUpdateTable t; // base strideOp strideUnit minResults - t["pto.vlds"] = { 0, 1, StrideUnit::Element, 1 }; - t["pto.vsts"] = { 1, 2, StrideUnit::Element, 0 }; - t["pto.vsstb"] = { 1, 3, StrideUnit::Block, 0 }; + t["pto.vlds"] = {0, 1, StrideUnit::Element, 1}; + t["pto.vsts"] = {1, 2, StrideUnit::Element, 0}; + t["pto.vsstb"] = {1, 3, StrideUnit::Block, 0}; return t; }(); return table; @@ -250,8 +251,163 @@ static StrideExprRef makeCast(Operation *castOp, StrideExprRef a) { return e; } -static void collectLeaves(const StrideExprRef &e, - SmallVectorImpl &out) { +// A canonical affine view of StrideExpr used for exact division and symbolic +// equality. Cast expressions are kept as typed atoms so materialization can +// still clone or eliminate the cast according to the final operand type. +struct AffineTerm { + StrideExprRef atom; + int64_t coeff; +}; + +struct AffineForm { + int64_t constant = 0; + SmallVector terms; +}; + +static bool sameAffineAtom(const StrideExprRef &a, const StrideExprRef &b) { + if (!a || !b || a->kind != b->kind) + return false; + if (a->kind == StrideExpr::Kind::Leaf) + return a->leaf == b->leaf; + if (a->kind != StrideExpr::Kind::Cast) + return false; + return a->castOp->getName() == b->castOp->getName() && + a->castOp->getOperand(0).getType() == + b->castOp->getOperand(0).getType() && + a->castOp->getResult(0).getType() == + b->castOp->getResult(0).getType() && + sameAffineAtom(a->lhs, b->lhs); +} + +static bool addAffineConstant(AffineForm &form, int64_t value) { + int64_t result; + if (llvm::AddOverflow(form.constant, value, result)) + return false; + form.constant = result; + return true; +} + +static bool addAffineTerm(AffineForm &form, StrideExprRef atom, int64_t coeff) { + if (coeff == 0) + return true; + for (unsigned i = 0; i < form.terms.size(); ++i) { + if (!sameAffineAtom(form.terms[i].atom, atom)) + continue; + int64_t result; + if (llvm::AddOverflow(form.terms[i].coeff, coeff, result)) + return false; + if (result == 0) + form.terms.erase(form.terms.begin() + i); + else + form.terms[i].coeff = result; + return true; + } + form.terms.push_back({std::move(atom), coeff}); + return true; +} + +static bool accumulateAffine(const StrideExprRef &e, int64_t scale, + AffineForm &form) { + if (!e) + return false; + switch (e->kind) { + case StrideExpr::Kind::Const: { + int64_t scaled; + return !llvm::MulOverflow(e->constant, scale, scaled) && + addAffineConstant(form, scaled); + } + case StrideExpr::Kind::Leaf: { + if (auto c = getConstantIntValue(e->leaf)) { + int64_t scaled; + return !llvm::MulOverflow(*c, scale, scaled) && + addAffineConstant(form, scaled); + } + return addAffineTerm(form, e, scale); + } + case StrideExpr::Kind::Cast: + if (auto c = foldConst(e)) { + int64_t scaled; + return !llvm::MulOverflow(*c, scale, scaled) && + addAffineConstant(form, scaled); + } + return addAffineTerm(form, e, scale); + case StrideExpr::Kind::Add: + return accumulateAffine(e->lhs, scale, form) && + accumulateAffine(e->rhs, scale, form); + case StrideExpr::Kind::Sub: { + int64_t negScale; + return !llvm::SubOverflow(int64_t{0}, scale, negScale) && + accumulateAffine(e->lhs, scale, form) && + accumulateAffine(e->rhs, negScale, form); + } + case StrideExpr::Kind::Mul: { + if (auto lhsConst = foldConst(e->lhs)) { + int64_t newScale; + return !llvm::MulOverflow(scale, *lhsConst, newScale) && + accumulateAffine(e->rhs, newScale, form); + } + if (auto rhsConst = foldConst(e->rhs)) { + int64_t newScale; + return !llvm::MulOverflow(scale, *rhsConst, newScale) && + accumulateAffine(e->lhs, newScale, form); + } + return false; + } + } + return false; +} + +static std::optional normalizeAffine(const StrideExprRef &e) { + AffineForm form; + if (!accumulateAffine(e, 1, form)) + return std::nullopt; + return form; +} + +static bool equalAffineForms(const AffineForm &a, const AffineForm &b) { + if (a.constant != b.constant || a.terms.size() != b.terms.size()) + return false; + for (const AffineTerm &termA : a.terms) { + bool found = llvm::any_of(b.terms, [&](const AffineTerm &termB) { + return termA.coeff == termB.coeff && + sameAffineAtom(termA.atom, termB.atom); + }); + if (!found) + return false; + } + return true; +} + +static bool isZeroAffineForm(const AffineForm &form) { + return form.constant == 0 && form.terms.empty(); +} + +static bool divideAffineForm(AffineForm &form, int64_t divisor) { + if (divisor <= 0 || form.constant % divisor != 0) + return false; + for (const AffineTerm &term : form.terms) + if (term.coeff % divisor != 0) + return false; + form.constant /= divisor; + for (AffineTerm &term : form.terms) + term.coeff /= divisor; + return true; +} + +static StrideExprRef affineFormToExpr(const AffineForm &form) { + StrideExprRef result; + if (form.constant != 0) + result = makeConst(form.constant); + for (const AffineTerm &term : form.terms) { + StrideExprRef value = term.atom; + if (term.coeff != 1) + value = makeMul(value, makeConst(term.coeff)); + result = result ? makeAdd(result, value) : value; + } + return result ? result : makeConst(0); +} + +static void collectLeaves(const StrideExprRef &e, SmallVectorImpl &out) { if (!e) return; if (e->kind == StrideExpr::Kind::Leaf) { @@ -301,9 +457,10 @@ using DecompCache = DenseMap>; // Decompose `v` into blockArg * coeff + increment by recursing through // addi/subi/muli/index_cast/addptr chains. Pure: builds only StrideExprs. // `cache` is scoped to one decomposition (a single `blockArg`). -static std::optional -decomposeLinear(Value v, BlockArgument blockArg, scf::ForOp forOp, - DecompCache &cache) { +static std::optional decomposeLinear(Value v, + BlockArgument blockArg, + scf::ForOp forOp, + DecompCache &cache) { // v == blockArg → {1, 0} if (v == blockArg) return LinearDecomp{1, makeConst(0)}; @@ -336,10 +493,10 @@ decomposeLinear(Value v, BlockArgument blockArg, scf::ForOp forOp, if (da->coeff == 0 && db->coeff == 0) return record(LinearDecomp{0, makeLeaf(v)}); bool isSub = isa(defOp); - return record(LinearDecomp{ - isSub ? da->coeff - db->coeff : da->coeff + db->coeff, - isSub ? makeSub(da->increment, db->increment) - : makeAdd(da->increment, db->increment)}); + return record( + LinearDecomp{isSub ? da->coeff - db->coeff : da->coeff + db->coeff, + isSub ? makeSub(da->increment, db->increment) + : makeAdd(da->increment, db->increment)}); } // v = muli(a, b), one side blockArg-free with constant k → {c * k, i * k} @@ -357,8 +514,9 @@ decomposeLinear(Value v, BlockArgument blockArg, scf::ForOp forOp, auto constMul = getConstantIntValue(multiplier); if (!constMul) return record(std::nullopt); - return record(LinearDecomp{withBA.coeff * *constMul, - makeMul(withBA.increment, makeConst(*constMul))}); + return record( + LinearDecomp{withBA.coeff * *constMul, + makeMul(withBA.increment, makeConst(*constMul))}); } // v = index_cast(a) → {ca, cast(ia)} @@ -496,8 +654,7 @@ static StrideExprRef computeDelta(Value v, scf::ForOp forOp, // Block argument from iter_args: check yield = arg + c if (auto blockArg = dyn_cast(v)) { - if (blockArg.getOwner() == forOp.getBody() && - blockArg.getArgNumber() > 0) { + if (blockArg.getOwner() == forOp.getBody() && blockArg.getArgNumber() > 0) { unsigned idx = blockArg.getArgNumber() - 1; auto yieldOp = cast(forOp.getBody()->getTerminator()); Value yieldVal = yieldOp.getOperand(idx); @@ -593,8 +750,7 @@ static Value materializeAtLoopEntry(Value v, scf::ForOp forOp, // iter_arg → its init value if (auto blockArg = dyn_cast(v)) { - if (blockArg.getOwner() == forOp.getBody() && - blockArg.getArgNumber() > 0) { + if (blockArg.getOwner() == forOp.getBody() && blockArg.getArgNumber() > 0) { unsigned idx = blockArg.getArgNumber() - 1; return forOp.getInitArgs()[idx]; } @@ -625,14 +781,54 @@ static Value materializeAtLoopEntry(Value v, scf::ForOp forOp, return cloned->getResult(cast(v).getResultNumber()); } -// Compute the initial pointer, i.e. the address the op reaches on the first -// iteration: base_at_iter0 + strideOperand_at_iter0 restated in addptr units. -// -// This is the mirror of the rescaling in `scaleBaseDelta`. The initial offset -// is W * strideOperand_0 bytes, and `pto.addptr` takes elements, so the offset -// it needs is (W/E) * strideOperand_0. For Element-class ops W == E and the -// operand is used as-is; for Block-class ops the factor W/E is exact (E always -// divides 32); for Byte-class ops the factor is a division that can fail. +// Whether strideOperand can be restated in pto.addptr element units without +// creating IR. Element and block units are always exact for supported element +// types; finer byte units require a suitably aligned compile-time constant. +static bool canScaleInitialOffset(Value strideOperand, int64_t elemBytes, + int64_t unitBytes) { + if (!strideOperand || unitBytes == elemBytes || unitBytes % elemBytes == 0) + return true; + if (elemBytes % unitBytes != 0) + return false; + auto constant = getConstantIntValue(strideOperand); + return constant && *constant % (elemBytes / unitBytes) == 0; +} + +// Create the address reached by one memory op before post-update rewriting. +// The builder must already point at the desired insertion location. +static Value createInitialPtr(Value base, Value strideOperand, + int64_t elemBytes, int64_t unitBytes, + Location loc, OpBuilder &builder) { + if (!strideOperand) + return base; + auto constSo = getConstantIntValue(strideOperand); + if (constSo && *constSo == 0) + return base; + if (!canScaleInitialOffset(strideOperand, elemBytes, unitBytes)) + return nullptr; + + Value scaledOffset = strideOperand; + if (unitBytes != elemBytes) { + if (unitBytes % elemBytes == 0) { + Value soIndex = strideOperand; + if (strideOperand.getType() != builder.getIndexType()) + soIndex = builder.create( + loc, builder.getIndexType(), strideOperand); + Value factor = + builder.create(loc, unitBytes / elemBytes); + scaledOffset = builder.create(loc, soIndex, factor); + } else { + int64_t divisor = elemBytes / unitBytes; + scaledOffset = + builder.create(loc, *constSo / divisor); + } + } + return builder.create(loc, base, scaledOffset); +} + +// Compute the initial pointer for a loop candidate, i.e. the address reached on +// the first iteration. Values defined in the loop are first materialized at the +// loop entry, then the shared unit conversion above is applied. static Value computeInitialPtr(Value base, Value strideOperand, int64_t elemBytes, int64_t unitBytes, scf::ForOp forOp, OpBuilder &builder) { @@ -647,37 +843,9 @@ static Value computeInitialPtr(Value base, Value strideOperand, if (!soAtEntry) return nullptr; - auto constSo = getConstantIntValue(soAtEntry); - if (constSo && *constSo == 0) - return baseAtEntry; - builder.setInsertionPoint(forOp); - Value scaledOffset = soAtEntry; - if (unitBytes != elemBytes) { - if (unitBytes % elemBytes == 0) { - Value soIndex = soAtEntry; - if (soAtEntry.getType() != builder.getIndexType()) - soIndex = builder.create( - forOp.getLoc(), builder.getIndexType(), soAtEntry); - Value factor = builder.create( - forOp.getLoc(), unitBytes / elemBytes); - scaledOffset = - builder.create(forOp.getLoc(), soIndex, factor); - } else if (elemBytes % unitBytes == 0) { - // Finer stride unit than the pointer element (a raw byte offset on a - // wider element type): only representable when it lands on an element - // boundary, which we can only prove for a compile-time constant. - int64_t divisor = elemBytes / unitBytes; - if (!constSo || *constSo % divisor != 0) - return nullptr; - scaledOffset = builder.create( - forOp.getLoc(), *constSo / divisor); - } else { - return nullptr; - } - } - return builder.create(forOp.getLoc(), baseAtEntry, - scaledOffset); + return createInitialPtr(baseAtEntry, soAtEntry, elemBytes, unitBytes, + forOp.getLoc(), builder); } // Rescale a per-iteration base delta from `pto.addptr` units (elements) into @@ -698,12 +866,12 @@ static StrideExprRef scaleBaseDelta(StrideExprRef deltaBase, int64_t elemBytes, if (unitBytes % elemBytes == 0) { // Coarser stride unit (e.g. 32-byte blocks over 4-byte elements): the base - // delta must be a compile-time constant that lands on a whole unit. + // delta's affine coefficients must all land on a whole unit. int64_t divisor = unitBytes / elemBytes; - auto constBase = foldConst(deltaBase); - if (!constBase || *constBase % divisor != 0) + auto form = normalizeAffine(deltaBase); + if (!form || !divideAffineForm(*form, divisor)) return nullptr; - return makeConst(*constBase / divisor); + return affineFormToExpr(*form); } if (elemBytes % unitBytes == 0) @@ -721,12 +889,16 @@ static StrideExprRef scaleBaseDelta(StrideExprRef deltaBase, int64_t elemBytes, static StrideExprRef combineStride(StrideExprRef deltaBase, StrideExprRef deltaOffset, int64_t elemBytes, int64_t unitBytes) { - StrideExprRef scaledBase = - scaleBaseDelta(deltaBase, elemBytes, unitBytes); + StrideExprRef scaledBase = scaleBaseDelta(deltaBase, elemBytes, unitBytes); if (!scaledBase) return nullptr; StrideExprRef total = makeAdd(scaledBase, deltaOffset); + if (auto form = normalizeAffine(total)) { + if (isZeroAffineForm(*form)) + return nullptr; + return affineFormToExpr(*form); + } if (auto constTotal = foldConst(total); constTotal && *constTotal == 0) return nullptr; return total; @@ -908,6 +1080,138 @@ static Value materialize(const StrideExprRef &e, Type wantType, Location loc, llvm_unreachable("unhandled StrideExpr kind"); } +// Sequential runs use the operand type fixed by the op definition. A cast may +// be omitted when materializing back to its input type; this is what lets a +// block-stride derived from `8 * index_cast(%i16)` become `%i16`. +static bool canMaterializeAs(const StrideExprRef &e, Type wantType) { + switch (e->kind) { + case StrideExpr::Kind::Const: + return true; + case StrideExpr::Kind::Leaf: + return e->leaf.getType() == wantType; + case StrideExpr::Kind::Cast: { + Type inputType = e->castOp->getOperand(0).getType(); + Type resultType = e->castOp->getResult(0).getType(); + if (wantType == inputType) + return canMaterializeAs(e->lhs, inputType); + return wantType == resultType && canMaterializeAs(e->lhs, inputType); + } + case StrideExpr::Kind::Add: + case StrideExpr::Kind::Sub: + case StrideExpr::Kind::Mul: + return canMaterializeAs(e->lhs, wantType) && + canMaterializeAs(e->rhs, wantType); + } + return false; +} + +// Dominance-aware counterpart of the loop-specific availability check above. +// Values already dominating the run head are reused; later pure definitions in +// the same block may be cloned before it. +static bool canHoistBefore(Value v, Operation *insertPt, + DominanceInfo &dominance, + DenseMap &memo) { + if (dominance.dominates(v, insertPt)) + return true; + auto it = memo.find(v); + if (it != memo.end()) + return it->second; + memo[v] = false; + + Operation *defOp = v.getDefiningOp(); + if (!defOp || defOp->getBlock() != insertPt->getBlock() || !isPure(defOp)) + return false; + for (Value operand : defOp->getOperands()) + if (!canHoistBefore(operand, insertPt, dominance, memo)) + return false; + memo[v] = true; + return true; +} + +static Value hoistBefore(Value v, Operation *insertPt, DominanceInfo &dominance, + OpBuilder &builder, DenseMap &memo) { + if (dominance.dominates(v, insertPt)) + return v; + if (auto it = memo.find(v); it != memo.end()) + return it->second; + + Operation *defOp = v.getDefiningOp(); + SmallVector newOperands; + for (Value operand : defOp->getOperands()) + newOperands.push_back( + hoistBefore(operand, insertPt, dominance, builder, memo)); + + builder.setInsertionPoint(insertPt); + Operation *cloned = builder.clone(*defOp); + for (auto [i, operand] : llvm::enumerate(newOperands)) + cloned->setOperand(i, operand); + Value result = cloned->getResult(cast(v).getResultNumber()); + memo[v] = result; + return result; +} + +static StrideExprRef makeAvailableAt(const StrideExprRef &e, + Operation *insertPt, + DominanceInfo &dominance, + OpBuilder &builder, + DenseMap &memo) { + switch (e->kind) { + case StrideExpr::Kind::Const: + return e; + case StrideExpr::Kind::Leaf: { + Value available = hoistBefore(e->leaf, insertPt, dominance, builder, memo); + return available == e->leaf ? e : makeLeaf(available); + } + case StrideExpr::Kind::Cast: + return makeCast( + e->castOp, makeAvailableAt(e->lhs, insertPt, dominance, builder, memo)); + case StrideExpr::Kind::Add: + return makeAdd(makeAvailableAt(e->lhs, insertPt, dominance, builder, memo), + makeAvailableAt(e->rhs, insertPt, dominance, builder, memo)); + case StrideExpr::Kind::Sub: + return makeSub(makeAvailableAt(e->lhs, insertPt, dominance, builder, memo), + makeAvailableAt(e->rhs, insertPt, dominance, builder, memo)); + case StrideExpr::Kind::Mul: + return makeMul(makeAvailableAt(e->lhs, insertPt, dominance, builder, memo), + makeAvailableAt(e->rhs, insertPt, dominance, builder, memo)); + } + llvm_unreachable("unhandled StrideExpr kind"); +} + +static Value materializeSequential(const StrideExprRef &e, Type wantType, + Location loc, OpBuilder &builder) { + switch (e->kind) { + case StrideExpr::Kind::Const: + if (wantType.isIndex()) + return builder.create(loc, e->constant); + return builder.create( + loc, e->constant, wantType.getIntOrFloatBitWidth()); + case StrideExpr::Kind::Leaf: + return e->leaf; + case StrideExpr::Kind::Cast: { + Type inputType = e->castOp->getOperand(0).getType(); + if (wantType == inputType) + return materializeSequential(e->lhs, inputType, loc, builder); + Value input = materializeSequential(e->lhs, inputType, loc, builder); + Operation *cloned = builder.clone(*e->castOp); + cloned->setOperand(0, input); + return cloned->getResult(0); + } + case StrideExpr::Kind::Add: + case StrideExpr::Kind::Sub: + case StrideExpr::Kind::Mul: { + Value lhs = materializeSequential(e->lhs, wantType, loc, builder); + Value rhs = materializeSequential(e->rhs, wantType, loc, builder); + if (e->kind == StrideExpr::Kind::Add) + return builder.create(loc, lhs, rhs); + if (e->kind == StrideExpr::Kind::Sub) + return builder.create(loc, lhs, rhs); + return builder.create(loc, lhs, rhs); + } + } + llvm_unreachable("unhandled StrideExpr kind"); +} + // Information about a post-update transformation to apply. struct PostUpdateRewrite { Operation *op; @@ -937,11 +1241,31 @@ static IterArgGroupKey getGroupKey(const PostUpdateRewrite &rw) { return {rw.base, rw.strideOperand, rw.stride}; } +// Build the post-update form of an op while preserving every operand, +// attribute, and original result. The updated base is always appended last. +static Operation *createPostUpdateOp(Operation *op, + const PostUpdateOpInfo &info, Value base, + Value stride, OpBuilder &builder) { + OperationState state(op->getLoc(), op->getName()); + for (auto [i, operand] : llvm::enumerate(op->getOperands())) { + if (static_cast(i) == info.baseOperandIdx) + state.addOperands(base); + else if (static_cast(i) == info.strideOperandIdx) + state.addOperands(stride); + else + state.addOperands(operand); + } + state.addTypes(op->getResultTypes()); + state.addTypes(base.getType()); + state.addAttributes(op->getAttrs()); + return builder.create(state); +} + // Apply post-update rewrites to a single scf.for. // Returns the new ForOp if any rewrites were applied, null otherwise. -static scf::ForOp applyPostUpdateRewrites( - scf::ForOp forOp, ArrayRef rewrites, - OpBuilder &builder) { +static scf::ForOp applyPostUpdateRewrites(scf::ForOp forOp, + ArrayRef rewrites, + OpBuilder &builder) { if (rewrites.empty()) return nullptr; @@ -951,7 +1275,8 @@ static scf::ForOp applyPostUpdateRewrites( // + vsts both accessing %base[%iv]). DenseMap groupToIdx; // group key -> iter_arg index SmallVector rwGroupIdx(rewrites.size()); // rewrite -> group index - SmallVector groupInitPtrs; // initial pointer per group (base + offset_at_iter0) + SmallVector + groupInitPtrs; // initial pointer per group (base + offset_at_iter0) for (auto [i, rw] : llvm::enumerate(rewrites)) { auto key = getGroupKey(rw); @@ -965,7 +1290,7 @@ static scf::ForOp applyPostUpdateRewrites( // Build new init args: original + one new pointer per group. SmallVector newInitArgs(forOp.getInitArgs().begin(), - forOp.getInitArgs().end()); + forOp.getInitArgs().end()); for (Value ptr : groupInitPtrs) newInitArgs.push_back(ptr); @@ -1015,23 +1340,8 @@ static scf::ForOp applyPostUpdateRewrites( if (!info) continue; - // Build the post-update op generically: replace base and strideOperand, - // keep all other operands, append updated_base to result types. - OperationState state(clonedOp->getLoc(), clonedOp->getName()); - for (auto [i, operand] : llvm::enumerate(clonedOp->getOperands())) { - if (static_cast(i) == info->baseOperandIdx) - state.addOperands(ptr); - else if (static_cast(i) == info->strideOperandIdx) - state.addOperands(strideNew); - else - state.addOperands(operand); - } - for (Type t : clonedOp->getResultTypes()) - state.addTypes(t); - state.addTypes(ptr.getType()); // updated_base (appended last) - state.addAttributes(clonedOp->getAttrs()); - - Operation *newOp = builder.create(state); + Operation *newOp = + createPostUpdateOp(clonedOp, *info, ptr, strideNew, builder); // Replace old results with new and update the mapping so that later // yield construction via mapping.lookupOrDefault sees the new results @@ -1065,6 +1375,269 @@ static scf::ForOp applyPostUpdateRewrites( return newForOp; } +//===----------------------------------------------------------------------===// +// Sequential Path +//===----------------------------------------------------------------------===// + +using SequentialExprCache = DenseMap; + +// Build an affine StrideExpr for one scalar address operand. Unsupported +// arithmetic remains an opaque leaf, so it may still cancel when the exact same +// SSA value is reused without guessing at its semantics. +static StrideExprRef buildSequentialExpr(Value value, + SequentialExprCache &cache) { + if (auto constant = getConstantIntValue(value)) + return makeConst(*constant); + if (auto it = cache.find(value); it != cache.end()) + return it->second; + + Operation *defOp = value.getDefiningOp(); + StrideExprRef result; + if (auto add = dyn_cast_or_null(defOp)) { + result = makeAdd(buildSequentialExpr(add.getLhs(), cache), + buildSequentialExpr(add.getRhs(), cache)); + } else if (auto sub = dyn_cast_or_null(defOp)) { + result = makeSub(buildSequentialExpr(sub.getLhs(), cache), + buildSequentialExpr(sub.getRhs(), cache)); + } else if (auto mul = dyn_cast_or_null(defOp)) { + if (auto lhsConst = getConstantIntValue(mul.getLhs())) + result = makeMul(makeConst(*lhsConst), + buildSequentialExpr(mul.getRhs(), cache)); + else if (auto rhsConst = getConstantIntValue(mul.getRhs())) + result = makeMul(buildSequentialExpr(mul.getLhs(), cache), + makeConst(*rhsConst)); + else + result = makeLeaf(value); + } else if (isa_and_nonnull(defOp)) { + result = makeCast(defOp, buildSequentialExpr(defOp->getOperand(0), cache)); + } else { + result = makeLeaf(value); + } + cache[value] = result; + return result; +} + +struct NormalizedBase { + Value root; + StrideExprRef offset; // in pto.addptr element units +}; + +// Strip a same-element-unit pto.addptr chain into (root, accumulated offset). +static NormalizedBase normalizeSequentialBase(Value base, int64_t elemBytes, + SequentialExprCache &cache) { + Value root = base; + StrideExprRef offset = makeConst(0); + while (auto addPtr = root.getDefiningOp()) { + auto parentElemBytes = addPtrUnitBytes(addPtr.getPtr()); + if (!parentElemBytes || *parentElemBytes != elemBytes) + break; + offset = makeAdd(offset, buildSequentialExpr(addPtr.getOffset(), cache)); + root = addPtr.getPtr(); + } + return {root, offset}; +} + +struct SequentialCandidate { + Operation *op; + const PostUpdateOpInfo *info; + Value base; + Value strideOperand; + Value rootBase; + StrideExprRef baseOffset; + StrideExprRef strideExpr; + int64_t elemBytes; + int64_t unitBytes; +}; + +struct SequentialBucket { + StringRef opName; + Value rootBase; + SmallVector candidates; +}; + +struct SequentialStep { + StrideExprRef expr; + AffineForm form; +}; + +static std::optional +analyzeSequentialStep(const SequentialCandidate &previous, + const SequentialCandidate ¤t) { + if (previous.elemBytes != current.elemBytes || + previous.unitBytes != current.unitBytes) + return std::nullopt; + + StrideExprRef deltaBase = makeSub(current.baseOffset, previous.baseOffset); + StrideExprRef deltaStride = makeSub(current.strideExpr, previous.strideExpr); + StrideExprRef step = combineStride(deltaBase, deltaStride, current.elemBytes, + current.unitBytes); + if (!step) + return std::nullopt; + auto form = normalizeAffine(step); + if (!form || isZeroAffineForm(*form)) + return std::nullopt; + return SequentialStep{affineFormToExpr(*form), std::move(*form)}; +} + +struct SequentialRun { + SmallVector candidates; + StrideExprRef step; + AffineForm stepForm; + Type strideType; + Value strideValue; + Value currentPtr; +}; + +static bool validateSequentialRun(SequentialRun &run, + DominanceInfo &dominance) { + if (run.candidates.size() < 3) + return false; + + SequentialCandidate *first = run.candidates.front(); + run.strideType = first->strideOperand.getType(); + if (!canMaterializeAs(run.step, run.strideType) || + !constantsFitType(run.step, run.strideType) || + !canScaleInitialOffset(first->strideOperand, first->elemBytes, + first->unitBytes)) + return false; + + for (SequentialCandidate *candidate : run.candidates) + if (candidate->strideOperand.getType() != run.strideType) + return false; + + SmallVector leaves; + collectLeaves(run.step, leaves); + DenseMap canCache; + return llvm::all_of(leaves, [&](Value leaf) { + return canHoistBefore(leaf, first->op, dominance, canCache); + }); +} + +static void collectNestedBlocks(Operation *op, pto::VecScopeOp owner, + SmallVectorImpl &blocks) { + for (Region ®ion : op->getRegions()) { + for (Block &block : region) { + blocks.push_back(&block); + for (Operation &nested : block) { + if (auto nestedScope = dyn_cast(nested); + nestedScope && nestedScope != owner) + continue; + collectNestedBlocks(&nested, owner, blocks); + } + } + } +} + +static void processSequentialBlock(Block *block, DominanceInfo &dominance, + OpBuilder &builder) { + SmallVector originalOps; + SmallVector buckets; + SequentialExprCache exprCache; + + for (Operation &op : *block) { + originalOps.push_back(&op); + const PostUpdateOpInfo *info = getPostUpdateInfo(&op); + if (!info || isAlreadyPostUpdate(&op, *info)) + continue; + + Value base, strideOperand; + extractBaseAndStrideOperand(&op, *info, base, strideOperand); + auto elemBytes = addPtrUnitBytes(base); + if (!elemBytes) + continue; + int64_t unitBytes = strideUnitBytes(info->strideUnit, *elemBytes); + NormalizedBase normalized = + normalizeSequentialBase(base, *elemBytes, exprCache); + + auto bucketIt = llvm::find_if(buckets, [&](const SequentialBucket &bucket) { + return bucket.opName == op.getName().getStringRef() && + bucket.rootBase == normalized.root; + }); + if (bucketIt == buckets.end()) { + buckets.push_back({op.getName().getStringRef(), normalized.root, {}}); + bucketIt = std::prev(buckets.end()); + } + bucketIt->candidates.push_back( + {&op, info, base, strideOperand, normalized.root, normalized.offset, + buildSequentialExpr(strideOperand, exprCache), *elemBytes, unitBytes}); + } + + SmallVector runs; + for (SequentialBucket &bucket : buckets) { + auto &candidates = bucket.candidates; + size_t start = 0; + while (start + 1 < candidates.size()) { + auto firstStep = + analyzeSequentialStep(candidates[start], candidates[start + 1]); + if (!firstStep) { + ++start; + continue; + } + + size_t end = start + 2; + while (end < candidates.size()) { + auto nextStep = + analyzeSequentialStep(candidates[end - 1], candidates[end]); + if (!nextStep || !equalAffineForms(firstStep->form, nextStep->form)) + break; + ++end; + } + + SequentialRun run; + run.step = firstStep->expr; + run.stepForm = firstStep->form; + for (size_t i = start; i < end; ++i) + run.candidates.push_back(&candidates[i]); + if (validateSequentialRun(run, dominance)) + runs.push_back(std::move(run)); + + // Runs are deliberately non-overlapping. The candidate that broke the + // current stride becomes the start of the next run. + start = end; + } + } + + if (runs.empty()) + return; + + // Materialize every accepted run before erasing any candidate op. + for (SequentialRun &run : runs) { + SequentialCandidate *first = run.candidates.front(); + DenseMap hoistMemo; + StrideExprRef available = + makeAvailableAt(run.step, first->op, dominance, builder, hoistMemo); + builder.setInsertionPoint(first->op); + run.strideValue = materializeSequential(available, run.strideType, + first->op->getLoc(), builder); + builder.setInsertionPoint(first->op); + run.currentPtr = + createInitialPtr(first->base, first->strideOperand, first->elemBytes, + first->unitBytes, first->op->getLoc(), builder); + } + + DenseMap opToRun; + for (auto [runIdx, run] : llvm::enumerate(runs)) + for (SequentialCandidate *candidate : run.candidates) + opToRun[candidate->op] = runIdx; + + // Rewrite in original program order so interleaved buckets maintain separate + // pointer chains without invalidating one another. + for (Operation *op : originalOps) { + auto it = opToRun.find(op); + if (it == opToRun.end()) + continue; + SequentialRun &run = runs[it->second]; + const PostUpdateOpInfo *info = getPostUpdateInfo(op); + builder.setInsertionPoint(op); + Operation *newOp = + createPostUpdateOp(op, *info, run.currentPtr, run.strideValue, builder); + for (unsigned result = 0; result < op->getNumResults(); ++result) + op->getResult(result).replaceAllUsesWith(newOp->getResult(result)); + run.currentPtr = newOp->getResult(newOp->getNumResults() - 1); + op->erase(); + } +} + //===----------------------------------------------------------------------===// // Pass Implementation //===----------------------------------------------------------------------===// @@ -1078,9 +1651,8 @@ struct VPTOSoftPostUpdatePass ModuleOp module = getOperation(); OpBuilder builder(&getContext()); - module.walk([&](pto::VecScopeOp vecscope) { - processVecScope(vecscope, builder); - }); + module.walk( + [&](pto::VecScopeOp vecscope) { processVecScope(vecscope, builder); }); } private: @@ -1097,6 +1669,15 @@ struct VPTOSoftPostUpdatePass // already-collected inner ForOp handles dangling. for (scf::ForOp forOp : forOps) processForOp(forOp, builder); + + // Loop rewriting rebuilds ForOps, so collect blocks only after every loop + // handle has been consumed. This second phase includes loop bodies and + // handles only candidates that remain in non-post-update form. + SmallVector blocks; + collectNestedBlocks(vecscope, vecscope, blocks); + DominanceInfo dominance(vecscope->getParentOp()); + for (Block *block : blocks) + processSequentialBlock(block, dominance, builder); } void processForOp(scf::ForOp forOp, OpBuilder &builder) { diff --git a/test/lit/vpto/soft_postupdate_sequential-base-chain.pto b/test/lit/vpto/soft_postupdate_sequential-base-chain.pto new file mode 100644 index 000000000..f61df0833 --- /dev/null +++ b/test/lit/vpto/soft_postupdate_sequential-base-chain.pto @@ -0,0 +1,51 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: ptoas --pto-backend=vpto --pto-arch=a5 --enable-vpto-soft-postupdate --emit-vpto %s -o - | FileCheck %s + +// Base normalization walks multiple addptr levels. Both base components and +// the explicit offset vary, but their combined addresses are 0, 64, and 128. +// CHECK-LABEL: func.func @multilevel_base_chain +// CHECK: pto.vecscope +// CHECK: [[INIT:%[a-zA-Z0-9_]+]] = pto.addptr +// CHECK: %{{.*}}, [[P1:%[a-zA-Z0-9_]+]] = pto.vlds [[INIT]][%c64] +// CHECK: %{{.*}}, [[P2:%[a-zA-Z0-9_]+]] = pto.vlds [[P1]][%c64] +// CHECK: %{{.*}}, %{{.*}} = pto.vlds [[P2]][%c64] + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @multilevel_base_chain(%src: !pto.ptr, %sink: !pto.ptr, %x: index, %y: index) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c16 = arith.constant 16 : index + %c32 = arith.constant 32 : index + %c64 = arith.constant 64 : index + %x16 = arith.addi %x, %c16 : index + %y16 = arith.addi %y, %c16 : index + %x32 = arith.addi %x, %c32 : index + %y32 = arith.addi %y, %c32 : index + %sum = arith.addi %x, %y : index + %off0 = arith.subi %c0, %sum : index + %off1 = arith.subi %c32, %sum : index + %off2 = arith.subi %c64, %sum : index + %base0a = pto.addptr %src, %x : !pto.ptr -> !pto.ptr + %base0 = pto.addptr %base0a, %y : !pto.ptr -> !pto.ptr + %base1a = pto.addptr %src, %x16 : !pto.ptr -> !pto.ptr + %base1 = pto.addptr %base1a, %y16 : !pto.ptr -> !pto.ptr + %base2a = pto.addptr %src, %x32 : !pto.ptr -> !pto.ptr + %base2 = pto.addptr %base2a, %y32 : !pto.ptr -> !pto.ptr + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %v0 = pto.vlds %base0[%off0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v1 = pto.vlds %base1[%off1] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v2 = pto.vlds %base2[%off2] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v2, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } +} diff --git a/test/lit/vpto/soft_postupdate_sequential-bucket-isolation.pto b/test/lit/vpto/soft_postupdate_sequential-bucket-isolation.pto new file mode 100644 index 000000000..d1d4fd489 --- /dev/null +++ b/test/lit/vpto/soft_postupdate_sequential-bucket-isolation.pto @@ -0,0 +1,71 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: ptoas --pto-backend=vpto --pto-arch=a5 --enable-vpto-soft-postupdate --emit-vpto %s -o - | FileCheck %s + +// Same-op candidates with different root bases form independent buckets even +// when their operations are physically interleaved. +// CHECK-LABEL: func.func @same_op_different_roots +// CHECK: %{{.*}}, [[A1:%[a-zA-Z0-9_]+]] = pto.vlds {{%.*}}[%c64] +// CHECK: %{{.*}}, [[B1:%[a-zA-Z0-9_]+]] = pto.vlds {{%.*}}[%c32] +// CHECK: %{{.*}}, [[A2:%[a-zA-Z0-9_]+]] = pto.vlds [[A1]][%c64] +// CHECK: %{{.*}}, [[B2:%[a-zA-Z0-9_]+]] = pto.vlds [[B1]][%c32] +// CHECK: %{{.*}}, %{{.*}} = pto.vlds [[A2]][%c64] +// CHECK: %{{.*}}, %{{.*}} = pto.vlds [[B2]][%c32] + +// An already-post-update op is excluded from candidate collection. It neither +// joins nor breaks the non-post run around it. +// CHECK-LABEL: func.func @already_post_does_not_break_run +// CHECK: %{{.*}}, [[P1:%[a-zA-Z0-9_]+]] = pto.vlds {{%.*}}[%c64] +// CHECK: %{{.*}}, %{{.*}} = pto.vlds {{%.*}}[%c4] +// CHECK: %{{.*}}, [[P2:%[a-zA-Z0-9_]+]] = pto.vlds [[P1]][%c64] +// CHECK: %{{.*}}, %{{.*}} = pto.vlds [[P2]][%c64] + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @same_op_different_roots(%src0: !pto.ptr, %src1: !pto.ptr, %sink: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c32 = arith.constant 32 : index + %c64 = arith.constant 64 : index + %c128 = arith.constant 128 : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %a0 = pto.vlds %src0[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %a0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %b0 = pto.vlds %src1[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %b0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %a1 = pto.vlds %src0[%c64] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %a1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %b1 = pto.vlds %src1[%c32] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %b1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %a2 = pto.vlds %src0[%c128] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %a2, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %b2 = pto.vlds %src1[%c64] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %b2, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } + + func.func @already_post_does_not_break_run(%src: !pto.ptr, %other: !pto.ptr, %sink: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c4 = arith.constant 4 : index + %c64 = arith.constant 64 : index + %c128 = arith.constant 128 : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %v0 = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %post, %next = pto.vlds %other[%c4] : !pto.ptr -> !pto.vreg<64xf32>, !pto.ptr + pto.vsts %post, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v1 = pto.vlds %src[%c64] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v2 = pto.vlds %src[%c128] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v2, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } +} diff --git a/test/lit/vpto/soft_postupdate_sequential-nested-blocks.pto b/test/lit/vpto/soft_postupdate_sequential-nested-blocks.pto new file mode 100644 index 000000000..8cad3e5fa --- /dev/null +++ b/test/lit/vpto/soft_postupdate_sequential-nested-blocks.pto @@ -0,0 +1,67 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: ptoas --pto-backend=vpto --pto-arch=a5 --enable-vpto-soft-postupdate --emit-vpto %s -o - | FileCheck %s + +// Sequential analysis visits nested scf.if blocks independently. +// CHECK-LABEL: func.func @sequential_scf_if +// CHECK: scf.if +// CHECK: %{{.*}}, [[I1:%[a-zA-Z0-9_]+]] = pto.vlds {{%.*}}[%c64] +// CHECK: %{{.*}}, [[I2:%[a-zA-Z0-9_]+]] = pto.vlds [[I1]][%c64] +// CHECK: %{{.*}}, %{{.*}} = pto.vlds [[I2]][%c64] + +// The IV access is handled by the loop path. The three constant accesses are +// left behind and then chained by the sequential path in the rebuilt loop. +// CHECK-LABEL: func.func @loop_partial_then_sequential +// CHECK: scf.for +// CHECK: %{{.*}}, %{{.*}} = pto.vlds {{%.*}}[{{%.*}}] +// CHECK: %{{.*}}, [[F1:%[a-zA-Z0-9_]+]] = pto.vlds {{%.*}}[%c64] +// CHECK: %{{.*}}, [[F2:%[a-zA-Z0-9_]+]] = pto.vlds [[F1]][%c64] +// CHECK: %{{.*}}, %{{.*}} = pto.vlds [[F2]][%c64] + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @sequential_scf_if(%src: !pto.ptr, %sink: !pto.ptr, %cond: i1) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c64 = arith.constant 64 : index + %c128 = arith.constant 128 : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + scf.if %cond { + %v0 = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v1 = pto.vlds %src[%c64] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v2 = pto.vlds %src[%c128] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v2, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + } + return + } + + func.func @loop_partial_then_sequential(%src: !pto.ptr, %sink: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c4 = arith.constant 4 : index + %c64 = arith.constant 64 : index + %c128 = arith.constant 128 : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + scf.for %iv = %c0 to %c4 step %c1 { + %loop = pto.vlds %src[%iv] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %loop, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v0 = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v1 = pto.vlds %src[%c64] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v2 = pto.vlds %src[%c128] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v2, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + } + return + } +} diff --git a/test/lit/vpto/soft_postupdate_sequential-run-restart.pto b/test/lit/vpto/soft_postupdate_sequential-run-restart.pto new file mode 100644 index 000000000..bbcd5865b --- /dev/null +++ b/test/lit/vpto/soft_postupdate_sequential-run-restart.pto @@ -0,0 +1,70 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: ptoas --pto-backend=vpto --pto-arch=a5 --enable-vpto-soft-postupdate --emit-vpto %s -o - | FileCheck %s + +// An invalid zero-step pair drops only its first candidate. The second +// candidate is reused as the head of a later three-op run. +// CHECK-LABEL: func.func @restart_after_zero_pair +// CHECK: {{^ *}}%{{[^, ]+}} = pto.vlds {{%.*}}[%c0] +// CHECK: %{{.*}}, [[Z1:%[a-zA-Z0-9_]+]] = pto.vlds {{%.*}}[%c64] +// CHECK: %{{.*}}, [[Z2:%[a-zA-Z0-9_]+]] = pto.vlds [[Z1]][%c64] +// CHECK: %{{.*}}, %{{.*}} = pto.vlds [[Z2]][%c64] + +// A valid pair that is too short is not rewritten. The candidate that breaks +// that tentative run becomes the head of the next run. +// CHECK-LABEL: func.func @restart_after_short_run +// CHECK: {{^ *}}%{{[^, ]+}} = pto.vlds {{%.*}}[%c0] +// CHECK: {{^ *}}%{{[^, ]+}} = pto.vlds {{%.*}}[%c64] +// CHECK: [[INIT:%[a-zA-Z0-9_]+]] = pto.addptr {{%.*}}, %c200 +// CHECK: %{{.*}}, [[S1:%[a-zA-Z0-9_]+]] = pto.vlds [[INIT]][%c32] +// CHECK: %{{.*}}, [[S2:%[a-zA-Z0-9_]+]] = pto.vlds [[S1]][%c32] +// CHECK: %{{.*}}, %{{.*}} = pto.vlds [[S2]][%c32] + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @restart_after_zero_pair(%src: !pto.ptr, %sink: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c64 = arith.constant 64 : index + %c128 = arith.constant 128 : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %v0 = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v1 = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v2 = pto.vlds %src[%c64] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v2, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v3 = pto.vlds %src[%c128] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v3, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } + + func.func @restart_after_short_run(%src: !pto.ptr, %sink: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c32 = arith.constant 32 : index + %c64 = arith.constant 64 : index + %c200 = arith.constant 200 : index + %c232 = arith.constant 232 : index + %c264 = arith.constant 264 : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %v0 = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v1 = pto.vlds %src[%c64] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v2 = pto.vlds %src[%c200] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v2, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v3 = pto.vlds %src[%c232] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v3, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v4 = pto.vlds %src[%c264] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v4, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } +} diff --git a/test/lit/vpto/soft_postupdate_sequential-symbolic-leaf.pto b/test/lit/vpto/soft_postupdate_sequential-symbolic-leaf.pto new file mode 100644 index 000000000..fa7bca386 --- /dev/null +++ b/test/lit/vpto/soft_postupdate_sequential-symbolic-leaf.pto @@ -0,0 +1,85 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: ptoas --pto-backend=vpto --pto-arch=a5 --enable-vpto-soft-postupdate --emit-vpto %s -o - | FileCheck %s + +// select is outside the supported affine grammar, but reusing the same SSA +// value makes it a stable symbolic leaf. Because it is defined after the run +// head, its pure definition is cloned before the first access. +// CHECK-LABEL: func.func @late_pure_symbolic_leaf +// CHECK: pto.vecscope +// CHECK: [[STEP:%[a-zA-Z0-9_]+]] = arith.select +// CHECK: %{{.*}}, [[L1:%[a-zA-Z0-9_]+]] = pto.vlds {{%.*}}[[[STEP]]] +// CHECK: %{{.*}}, [[L2:%[a-zA-Z0-9_]+]] = pto.vlds [[L1]][[[STEP]]] +// CHECK: %{{.*}}, %{{.*}} = pto.vlds [[L2]][[[STEP]]] + +// Distinct opaque SSA leaves are not assumed equal, so the candidate sequence +// is conservatively left in non-post-update form. +// CHECK-LABEL: func.func @distinct_symbolic_leaves +// CHECK: {{^ *}}%{{[^, ]+}} = pto.vlds {{%.*}}[%c0] +// CHECK: {{^ *}}%{{[^, ]+}} = pto.vlds {{%.*}}[{{%.*}}] +// CHECK: {{^ *}}%{{[^, ]+}} = pto.vlds {{%.*}}[{{%.*}}] + +// A potentially trapping definition cannot be moved before the run head. The +// run is rejected and no partial materialization is left behind. +// CHECK-LABEL: func.func @late_non_speculatable_leaf +// CHECK: {{^ *}}%{{[^, ]+}} = pto.vlds {{%.*}}[%c0] +// CHECK: arith.divui +// CHECK: {{^ *}}%{{[^, ]+}} = pto.vlds {{%.*}}[{{%.*}}] +// CHECK: {{^ *}}%{{[^, ]+}} = pto.vlds {{%.*}}[{{%.*}}] + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @late_pure_symbolic_leaf(%src: !pto.ptr, %sink: !pto.ptr, %cond: i1, %lhs: index, %rhs: index) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %v0 = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %step = arith.select %cond, %lhs, %rhs : index + %v1 = pto.vlds %src[%step] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %step2 = arith.addi %step, %step : index + %v2 = pto.vlds %src[%step2] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v2, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } + + func.func @distinct_symbolic_leaves(%src: !pto.ptr, %sink: !pto.ptr, %numerator: index, %denom0: index, %denom1: index) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %step0 = arith.divui %numerator, %denom0 : index + %step1 = arith.divui %numerator, %denom1 : index + %sum = arith.addi %step0, %step1 : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %v0 = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v1 = pto.vlds %src[%step0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v2 = pto.vlds %src[%sum] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v2, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } + + func.func @late_non_speculatable_leaf(%src: !pto.ptr, %sink: !pto.ptr, %numerator: index, %denominator: index) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %v0 = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %step = arith.divui %numerator, %denominator : index + %v1 = pto.vlds %src[%step] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %step2 = arith.addi %step, %step : index + %v2 = pto.vlds %src[%step2] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v2, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } +} diff --git a/test/lit/vpto/soft_postupdate_sequential-vsstb-initial.pto b/test/lit/vpto/soft_postupdate_sequential-vsstb-initial.pto new file mode 100644 index 000000000..c72787b10 --- /dev/null +++ b/test/lit/vpto/soft_postupdate_sequential-vsstb-initial.pto @@ -0,0 +1,60 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: ptoas --pto-backend=vpto --pto-arch=a5 --enable-vpto-soft-postupdate --emit-vpto %s -o - | FileCheck %s + +// The first repeat_stride is four 32-byte blocks. For f32 the initial pointer +// must therefore advance by 4 * (32 / 4) = 32 elements. +// CHECK-LABEL: func.func @vsstb_nonzero_initial_stride +// CHECK: [[INIT:%[a-zA-Z0-9_]+]] = pto.addptr {{%.*}}, %c32 +// CHECK: [[P1:%[a-zA-Z0-9_]+]] = pto.vsstb {{.*}}, [[INIT]], %c2_i16, %c3_i16, {{.*}} -> !pto.ptr !pto.ptr !pto.ptr{{ *$}} +// CHECK: {{^ *}}pto.vsstb {{.*}} : {{.*}}!pto.mask{{ *$}} +// CHECK: {{^ *}}pto.vsstb {{.*}} : {{.*}}!pto.mask{{ *$}} + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @vsstb_nonzero_initial_stride(%src: !pto.ptr, %dst: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c2_i16 = arith.constant 2 : i16 + %c4_i16 = arith.constant 4 : i16 + %c7_i16 = arith.constant 7 : i16 + %c10_i16 = arith.constant 10 : i16 + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %vec = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsstb %vec, %dst, %c2_i16, %c4_i16, %mask : !pto.vreg<64xf32>, !pto.ptr, i16, i16, !pto.mask + pto.vsstb %vec, %dst, %c2_i16, %c7_i16, %mask : !pto.vreg<64xf32>, !pto.ptr, i16, i16, !pto.mask + pto.vsstb %vec, %dst, %c2_i16, %c10_i16, %mask : !pto.vreg<64xf32>, !pto.ptr, i16, i16, !pto.mask + } + return + } + + func.func @vsstb_sequential_stride_out_of_range(%src: !pto.ptr, %dst: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c320000 = arith.constant 320000 : index + %c640000 = arith.constant 640000 : index + %c0_i16 = arith.constant 0 : i16 + %c2_i16 = arith.constant 2 : i16 + %base1 = pto.addptr %dst, %c320000 : !pto.ptr -> !pto.ptr + %base2 = pto.addptr %dst, %c640000 : !pto.ptr -> !pto.ptr + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %vec = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsstb %vec, %dst, %c2_i16, %c0_i16, %mask : !pto.vreg<64xf32>, !pto.ptr, i16, i16, !pto.mask + pto.vsstb %vec, %base1, %c2_i16, %c0_i16, %mask : !pto.vreg<64xf32>, !pto.ptr, i16, i16, !pto.mask + pto.vsstb %vec, %base2, %c2_i16, %c0_i16, %mask : !pto.vreg<64xf32>, !pto.ptr, i16, i16, !pto.mask + } + return + } +} diff --git a/test/lit/vpto/soft_postupdate_sequential-vsstb.pto b/test/lit/vpto/soft_postupdate_sequential-vsstb.pto new file mode 100644 index 000000000..7caa9c8a8 --- /dev/null +++ b/test/lit/vpto/soft_postupdate_sequential-vsstb.pto @@ -0,0 +1,65 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: ptoas --pto-backend=vpto --pto-arch=a5 --enable-vpto-soft-postupdate --emit-vpto %s -o - | FileCheck %s + +// For f32, a base delta of 8*k elements is exactly k 32-byte blocks. The +// index_cast is eliminated when the symbolic step is materialized as i16. +// CHECK-LABEL: func.func @sequential_vsstb_affine_block +// CHECK-SAME: ({{.*}}, {{.*}}, [[K:%[a-zA-Z0-9_]+]]: i16 +// CHECK: pto.vsstb {{.*}}, {{%.*}}, %c2_i16, [[K]], {{.*}} -> !pto.ptr !pto.ptr !pto.ptr{{ *$}} +// CHECK: {{^ *}}pto.vsstb {{.*}} : {{.*}}!pto.mask{{ *$}} +// CHECK: {{^ *}}pto.vsstb {{.*}} : {{.*}}!pto.mask{{ *$}} + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @sequential_vsstb_affine_block(%src: !pto.ptr, %dst: !pto.ptr, %k: i16) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c0_i16 = arith.constant 0 : i16 + %c2_i16 = arith.constant 2 : i16 + %k_idx = arith.index_castui %k : i16 to index + %c8 = arith.constant 8 : index + %c16 = arith.constant 16 : index + %off1 = arith.muli %k_idx, %c8 : index + %off2 = arith.muli %k_idx, %c16 : index + %base1 = pto.addptr %dst, %off1 : !pto.ptr -> !pto.ptr + %base2 = pto.addptr %dst, %off2 : !pto.ptr -> !pto.ptr + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %vec = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsstb %vec, %dst, %c2_i16, %c0_i16, %mask : !pto.vreg<64xf32>, !pto.ptr, i16, i16, !pto.mask + pto.vsstb %vec, %base1, %c2_i16, %c0_i16, %mask : !pto.vreg<64xf32>, !pto.ptr, i16, i16, !pto.mask + pto.vsstb %vec, %base2, %c2_i16, %c0_i16, %mask : !pto.vreg<64xf32>, !pto.ptr, i16, i16, !pto.mask + } + return + } + + func.func @sequential_vsstb_inexact_block(%src: !pto.ptr, %dst: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c0_i16 = arith.constant 0 : i16 + %c2_i16 = arith.constant 2 : i16 + %c4 = arith.constant 4 : index + %c8 = arith.constant 8 : index + %base1 = pto.addptr %dst, %c4 : !pto.ptr -> !pto.ptr + %base2 = pto.addptr %dst, %c8 : !pto.ptr -> !pto.ptr + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %vec = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsstb %vec, %dst, %c2_i16, %c0_i16, %mask : !pto.vreg<64xf32>, !pto.ptr, i16, i16, !pto.mask + pto.vsstb %vec, %base1, %c2_i16, %c0_i16, %mask : !pto.vreg<64xf32>, !pto.ptr, i16, i16, !pto.mask + pto.vsstb %vec, %base2, %c2_i16, %c0_i16, %mask : !pto.vreg<64xf32>, !pto.ptr, i16, i16, !pto.mask + } + return + } +} diff --git a/test/lit/vpto/soft_postupdate_sequential.pto b/test/lit/vpto/soft_postupdate_sequential.pto new file mode 100644 index 000000000..777608b01 --- /dev/null +++ b/test/lit/vpto/soft_postupdate_sequential.pto @@ -0,0 +1,217 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: ptoas --pto-backend=vpto --pto-arch=a5 --enable-vpto-soft-postupdate --emit-vpto %s -o - | FileCheck %s + +// Three fixed-base accesses form one post-update chain. +// CHECK-LABEL: func.func @sequential_constant +// CHECK: %{{.*}}, [[P1:%[a-zA-Z0-9_]+]] = pto.vlds {{%.*}}[{{%.*}}] {dist = "NORM"} +// CHECK: %{{.*}}, [[P2:%[a-zA-Z0-9_]+]] = pto.vlds [[P1]][{{%.*}}] {dist = "NORM"} +// CHECK: %{{.*}}, %{{.*}} = pto.vlds [[P2]][{{%.*}}] {dist = "NORM"} + +// A pair determines a step but does not meet the minimum run length. +// CHECK-LABEL: func.func @sequential_two_only +// CHECK: {{^ *}}%{{[^, ]+}} = pto.vlds {{%.*}}[%c0] +// CHECK: {{^ *}}%{{[^, ]+}} = pto.vlds {{%.*}}[%c64] + +// A zero-step sequence is not profitable and stays unchanged. +// CHECK-LABEL: func.func @sequential_zero_stride +// CHECK: {{^ *}}%{{[^, ]+}} = pto.vlds {{%.*}}[%c0] +// CHECK: {{^ *}}%{{[^, ]+}} = pto.vlds {{%.*}}[%c0] +// CHECK: {{^ *}}%{{[^, ]+}} = pto.vlds {{%.*}}[%c0] + +// The run step may be a loop-invariant affine value instead of a constant. +// CHECK-LABEL: func.func @sequential_affine_step +// CHECK-SAME: ({{.*}}, [[STEP:%[a-zA-Z0-9_]+]]: index +// CHECK: %{{.*}}, [[A1:%[a-zA-Z0-9_]+]] = pto.vlds {{%.*}}[[[STEP]]] +// CHECK: %{{.*}}, [[A2:%[a-zA-Z0-9_]+]] = pto.vlds [[A1]][[[STEP]]] +// CHECK: %{{.*}}, %{{.*}} = pto.vlds [[A2]][[[STEP]]] + +// Base and offset both vary, but their combined address step is fixed. +// CHECK-LABEL: func.func @sequential_base_offset_cancel +// CHECK: pto.vecscope +// CHECK: [[INIT:%[a-zA-Z0-9_]+]] = pto.addptr +// CHECK: %{{.*}}, [[C1:%[a-zA-Z0-9_]+]] = pto.vlds [[INIT]][{{%.*}}] +// CHECK: %{{.*}}, [[C2:%[a-zA-Z0-9_]+]] = pto.vlds [[C1]][{{%.*}}] +// CHECK: %{{.*}}, %{{.*}} = pto.vlds [[C2]][{{%.*}}] + +// Loop analysis cannot use the zero inter-iteration stride, so the sequential +// fallback still chains the accesses within one iteration. +// CHECK-LABEL: func.func @sequential_for_body +// CHECK: scf.for +// CHECK: %{{.*}}, [[F1:%[a-zA-Z0-9_]+]] = pto.vlds {{%.*}}[{{%.*}}] +// CHECK: %{{.*}}, [[F2:%[a-zA-Z0-9_]+]] = pto.vlds [[F1]][{{%.*}}] +// CHECK: %{{.*}}, %{{.*}} = pto.vlds [[F2]][{{%.*}}] + +// Different buckets may be physically interleaved. Each root keeps its own +// independent pointer chain. +// CHECK-LABEL: func.func @sequential_interleaved_buckets +// CHECK: %{{.*}}, [[X1:%[a-zA-Z0-9_]+]] = pto.vlds {{%.*}}[{{%.*}}] +// CHECK: [[Y1:%[a-zA-Z0-9_]+]] = pto.vsts {{.*}}, {{%.*}}[{{%.*}}], +// CHECK: %{{.*}}, [[X2:%[a-zA-Z0-9_]+]] = pto.vlds [[X1]][{{%.*}}] +// CHECK: [[Y2:%[a-zA-Z0-9_]+]] = pto.vsts {{.*}}, [[Y1]][{{%.*}}], +// CHECK: %{{.*}}, %{{.*}} = pto.vlds [[X2]][{{%.*}}] +// CHECK: %{{.*}} = pto.vsts {{.*}}, [[Y2]][{{%.*}}], + +// A stride break ends the first maximal run; the breaking candidate starts a +// second, non-overlapping run. +// CHECK-LABEL: func.func @sequential_maximal_runs +// CHECK: %{{.*}}, [[R1:%[a-zA-Z0-9_]+]] = pto.vlds {{%.*}}[{{%.*}}] +// CHECK: %{{.*}}, [[R2:%[a-zA-Z0-9_]+]] = pto.vlds [[R1]][{{%.*}}] +// CHECK: %{{.*}}, %{{.*}} = pto.vlds [[R2]][{{%.*}}] +// CHECK: [[INIT2:%[a-zA-Z0-9_]+]] = pto.addptr {{%.*}}, %c200 +// CHECK: %{{.*}}, [[R3:%[a-zA-Z0-9_]+]] = pto.vlds [[INIT2]][{{%.*}}] +// CHECK: %{{.*}}, [[R4:%[a-zA-Z0-9_]+]] = pto.vlds [[R3]][{{%.*}}] +// CHECK: %{{.*}}, %{{.*}} = pto.vlds [[R4]][{{%.*}}] + +module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { + func.func @sequential_constant(%src: !pto.ptr, %sink: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c64 = arith.constant 64 : index + %c128 = arith.constant 128 : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %v0 = pto.vlds %src[%c0] {dist = "NORM"} : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v1 = pto.vlds %src[%c64] {dist = "NORM"} : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v2 = pto.vlds %src[%c128] {dist = "NORM"} : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v2, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } + + func.func @sequential_two_only(%src: !pto.ptr, %sink: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c64 = arith.constant 64 : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %v0 = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v1 = pto.vlds %src[%c64] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } + + func.func @sequential_zero_stride(%src: !pto.ptr, %sink: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %v0 = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v1 = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v2 = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v2, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } + + func.func @sequential_affine_step(%src: !pto.ptr, %step: index, %sink: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %step2 = arith.addi %step, %step : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %v0 = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v1 = pto.vlds %src[%step] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v2 = pto.vlds %src[%step2] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v2, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } + + func.func @sequential_base_offset_cancel(%src: !pto.ptr, %x: index, %sink: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c16 = arith.constant 16 : index + %c32 = arith.constant 32 : index + %c48 = arith.constant 48 : index + %c96 = arith.constant 96 : index + %x16 = arith.addi %x, %c16 : index + %x32 = arith.addi %x, %c32 : index + %base0 = pto.addptr %src, %x : !pto.ptr -> !pto.ptr + %base1 = pto.addptr %src, %x16 : !pto.ptr -> !pto.ptr + %base2 = pto.addptr %src, %x32 : !pto.ptr -> !pto.ptr + %off0 = arith.subi %c0, %x : index + %off1 = arith.subi %c48, %x : index + %off2 = arith.subi %c96, %x : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %v0 = pto.vlds %base0[%off0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v1 = pto.vlds %base1[%off1] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v2 = pto.vlds %base2[%off2] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v2, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } + + func.func @sequential_for_body(%src: !pto.ptr, %sink: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c4 = arith.constant 4 : index + %c64 = arith.constant 64 : index + %c128 = arith.constant 128 : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + scf.for %iv = %c0 to %c4 step %c1 { + %v0 = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v1 = pto.vlds %src[%c64] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v2 = pto.vlds %src[%c128] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v2, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + } + return + } + + func.func @sequential_interleaved_buckets(%src: !pto.ptr, %dst: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c64 = arith.constant 64 : index + %c128 = arith.constant 128 : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %x0 = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %x0, %dst[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %x1 = pto.vlds %src[%c64] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %x1, %dst[%c64], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %x2 = pto.vlds %src[%c128] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %x2, %dst[%c128], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } + + func.func @sequential_maximal_runs(%src: !pto.ptr, %sink: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c64 = arith.constant 64 : index + %c128 = arith.constant 128 : index + %c200 = arith.constant 200 : index + %c232 = arith.constant 232 : index + %c264 = arith.constant 264 : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %v0 = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v1 = pto.vlds %src[%c64] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v2 = pto.vlds %src[%c128] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v2, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v3 = pto.vlds %src[%c200] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v3, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v4 = pto.vlds %src[%c232] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v4, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v5 = pto.vlds %src[%c264] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v5, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } +} From 5dbb06023d141be4095574dbf7b69c0f42ccbfc7 Mon Sep 17 00:00:00 2001 From: yexiaosu Date: Tue, 28 Jul 2026 11:17:13 +0800 Subject: [PATCH 2/3] Preserve i32 offsets in soft post-update --- lib/PTO/Transforms/VPTOSoftPostUpdate.cpp | 46 +++++++++++++++++------ 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/lib/PTO/Transforms/VPTOSoftPostUpdate.cpp b/lib/PTO/Transforms/VPTOSoftPostUpdate.cpp index 1095b3fdd..a9afca0d7 100644 --- a/lib/PTO/Transforms/VPTOSoftPostUpdate.cpp +++ b/lib/PTO/Transforms/VPTOSoftPostUpdate.cpp @@ -794,11 +794,29 @@ static bool canScaleInitialOffset(Value strideOperand, int64_t elemBytes, return constant && *constant % (elemBytes / unitBytes) == 0; } +// Reproduce the index-to-i32 truncation performed by vlds/vsts lowering before +// using an Element-class offset in pto.addptr. A direct index -> i32 -> index +// round trip is canonicalized away by arith, so route through i64 + trunci to +// keep the narrowing explicit. The final signed index_cast preserves negative +// offsets in the same way as the existing lowering. +static Value truncateElementOffsetToI32(Value offset, Location loc, + OpBuilder &builder) { + if (!offset.getType().isIndex()) + return offset; + Value offsetI64 = + builder.create(loc, builder.getI64Type(), offset); + Value offsetI32 = + builder.create(loc, builder.getI32Type(), offsetI64); + return builder.create(loc, builder.getIndexType(), + offsetI32); +} + // Create the address reached by one memory op before post-update rewriting. // The builder must already point at the desired insertion location. static Value createInitialPtr(Value base, Value strideOperand, - int64_t elemBytes, int64_t unitBytes, - Location loc, OpBuilder &builder) { + StrideUnit strideUnit, int64_t elemBytes, + int64_t unitBytes, Location loc, + OpBuilder &builder) { if (!strideOperand) return base; auto constSo = getConstantIntValue(strideOperand); @@ -807,7 +825,10 @@ static Value createInitialPtr(Value base, Value strideOperand, if (!canScaleInitialOffset(strideOperand, elemBytes, unitBytes)) return nullptr; - Value scaledOffset = strideOperand; + Value scaledOffset = + strideUnit == StrideUnit::Element + ? truncateElementOffsetToI32(strideOperand, loc, builder) + : strideOperand; if (unitBytes != elemBytes) { if (unitBytes % elemBytes == 0) { Value soIndex = strideOperand; @@ -830,8 +851,9 @@ static Value createInitialPtr(Value base, Value strideOperand, // the first iteration. Values defined in the loop are first materialized at the // loop entry, then the shared unit conversion above is applied. static Value computeInitialPtr(Value base, Value strideOperand, - int64_t elemBytes, int64_t unitBytes, - scf::ForOp forOp, OpBuilder &builder) { + StrideUnit strideUnit, int64_t elemBytes, + int64_t unitBytes, scf::ForOp forOp, + OpBuilder &builder) { Value baseAtEntry = materializeAtLoopEntry(base, forOp, builder); if (!baseAtEntry) return nullptr; @@ -844,8 +866,8 @@ static Value computeInitialPtr(Value base, Value strideOperand, return nullptr; builder.setInsertionPoint(forOp); - return createInitialPtr(baseAtEntry, soAtEntry, elemBytes, unitBytes, - forOp.getLoc(), builder); + return createInitialPtr(baseAtEntry, soAtEntry, strideUnit, elemBytes, + unitBytes, forOp.getLoc(), builder); } // Rescale a per-iteration base delta from `pto.addptr` units (elements) into @@ -1610,9 +1632,9 @@ static void processSequentialBlock(Block *block, DominanceInfo &dominance, run.strideValue = materializeSequential(available, run.strideType, first->op->getLoc(), builder); builder.setInsertionPoint(first->op); - run.currentPtr = - createInitialPtr(first->base, first->strideOperand, first->elemBytes, - first->unitBytes, first->op->getLoc(), builder); + run.currentPtr = createInitialPtr( + first->base, first->strideOperand, first->info->strideUnit, + first->elemBytes, first->unitBytes, first->op->getLoc(), builder); } DenseMap opToRun; @@ -1758,8 +1780,8 @@ struct VPTOSoftPostUpdatePass Value strideNew = materialize(finalExpr, strideType, op.getLoc(), forOp, constCache, builder); - Value initPtr = computeInitialPtr(base, strideOperand, *elemBytes, - unitBytes, forOp, builder); + Value initPtr = computeInitialPtr(base, strideOperand, info->strideUnit, + *elemBytes, unitBytes, forOp, builder); if (!initPtr) continue; From 3e10509bb5be40c89fda2d6bb14a6412439b3350 Mon Sep 17 00:00:00 2001 From: yexiaosu Date: Tue, 28 Jul 2026 15:36:22 +0800 Subject: [PATCH 3/3] Fix sequential run restart handling --- .../designs/vpto-soft-postupdate-design-zh.md | 85 +++++++++++++------ lib/PTO/Transforms/VPTOSoftPostUpdate.cpp | 14 +-- ...soft_postupdate_sequential-run-restart.pto | 28 ++++++ 3 files changed, 95 insertions(+), 32 deletions(-) diff --git a/docs/designs/vpto-soft-postupdate-design-zh.md b/docs/designs/vpto-soft-postupdate-design-zh.md index 9a1542d52..041da26c2 100644 --- a/docs/designs/vpto-soft-postupdate-design-zh.md +++ b/docs/designs/vpto-soft-postupdate-design-zh.md @@ -2,7 +2,7 @@ ## 1. 概述 -本文档设计一个新的 PTOAS pass——`VPTOSoftPostUpdate`,在 **MLIR 层**(LLVM lowering 之前)将非 Post-Update 形式的 VPTO 访存操作转换为 Post-Update 形式。首期目标指令为已支持 `updated_base` 的三个 op(`pto.vlds`、`pto.vsts`、`pto.vsstb`),后续扩展至更多访存指令(见第 2 节指令全景)。pass 覆盖两种场景:`scf.for` 循环间的固定步长访存模式(循环路径),以及同一 block 单次执行中的等步长访存 run(顺序路径)。 +本文档设计一个新的 PTOAS pass——`VPTOSoftPostUpdate`,在 **MLIR 层**(LLVM lowering 之前)将非 Post-Update 形式的 VPTO 访存操作转换为 Post-Update 形式。首期目标指令为已支持 `updated_base` 的三个 op(`pto.vlds`、`pto.vsts`、`pto.vsstb`),后续扩展至更多访存指令(见第 2 节指令全景)。pass 覆盖两种场景:`scf.for` 循环间的固定步长访存模式(循环路径),以及同一 block 单次执行中的 `SequentialRun`(顺序路径)。`SequentialRun` 指同一候选桶中按程序序连续、相邻候选的有效地址差均为同一个非零 `step` 的候选序列。 循环路径示例: @@ -125,7 +125,7 @@ pass 的驱动分为两个阶段。两个阶段都通过 `PostUpdateTable`(sta 阶段二 · 顺序路径:循环路径完成后,重新收集 vecscope 内的全部 block, 包括 scf.for body、vecscope 体和 scf.if 分支体。只扫描仍为非 Post-Update 形式的剩余 op;以 block 为单位分桶,桶内按程序序寻找互不重叠的 - maximal arithmetic runs,并链式改写。 + 最大 `SequentialRun`,并链式改写。 ``` 循环路径按内层到外层处理全部 `scf.for`。由于循环改写会 erase 并重建 ForOp,顺序路径必须在全部循环改写结束后重新收集 block 和候选 op,不能跨阶段保存 `Block *` 或 `Operation *`。这样循环路径已改写的 op 会因已有 `updated_base` 被自然跳过;循环路径未命中,以及原本位于 `scf.if` 等嵌套区域中的 op,仍可在单次 block 执行范围内参与顺序路径。 @@ -336,9 +336,9 @@ Post-Update 模式下 `repeat_stride` 从地址偏移变为指针前进量,因 ### 4.3 顺序路径 -顺序路径处理同一 block 单次执行中的等步长访问,包括非循环代码,以及 `scf.for` body 中循环路径未命中的剩余 op。步长只需在本次 block 执行期间保持不变,可以是编译期常量,也可以是由 block argument、循环 IV、iter_arg 或其他已存在 SSA 值组成的仿射表达式。 +顺序路径处理同一 block 单次执行中的等步长访问,包括非循环代码以及 `scf.for` body 中循环路径未命中的剩余 op。本节将同一 `(op 类型, rootBase)` 候选桶中按程序序连续、相邻候选的有效地址差均为同一个非零 `step` 的候选序列称为 `SequentialRun`。步长只需在本次 block 执行期间保持不变,可以是编译期常量,也可以是符号表达式。 -顺序路径复用循环路径的 `StrideExpr`、`scaleBaseDelta` / `combineStride`、类型与可用性检查、pure 定义链克隆、初始指针计算和 post-update op 构造。新增逻辑仅包括 base 归一化、仿射表达式规范化以及 block 内 run 检测。 +顺序路径复用循环路径的 `StrideExpr`、`scaleBaseDelta` / `combineStride`、类型与可用性检查、pure 定义链克隆、初始指针计算和 post-update op 构造。新增逻辑仅包括 base 归一化、以 SSA leaves 为变量的仿射规范化以及 block 内 `SequentialRun` 检测。 #### 4.3.1 有效地址分析 @@ -348,7 +348,26 @@ Post-Update 模式下 `repeat_stride` 从地址偏移变为指针前进量,因 base_i = pto.addptr(rootBase, baseOffset_i) // baseOffset_i 以元素计 ``` -若 base 不是 `pto.addptr` 的结果,则 `rootBase = base`、`baseOffset = 0`。`baseOffset` 和 strideOperand 沿与循环路径相同的 `arith.addi`、`arith.subi`、常量乘法和 index cast 规则构造成 `StrideExpr`;无法继续分解的 SSA 值作为叶子保留。 +- 若 base 不是 `pto.addptr` 的结果,则 `rootBase = base`、`baseOffset = 0`。 +- 若 base 来自 `pto.addptr`,则从它开始反向剥离连续的 `pto.addptr` 链:每经过一层就把该层 offset 累加到 `baseOffset`,并把该层输入指针作为新的待检查 root;直到 root 不再由 `pto.addptr` 定义,或下一层 `pto.addptr` 的元素单位与当前 base 不同。停止时的指针即 `rootBase`。因此只归一化元素单位一致的链,不跨越单位变化猜测换算关系。 + +例如: + +```mlir +%root = arith.select %cond, %lhs, %rhs : !pto.ptr +%p1 = pto.addptr %root, %a +%p2 = pto.addptr %p1, %b +%v = pto.vlds %p2[%c] : ... +``` + +在三者元素单位一致时归一化为: + +``` +rootBase = %root +baseOffset = %a + %b +``` + +随后将 `baseOffset` 与指令自身的 `%c` (strideOperand)一起用于有效地址 step 分析。`baseOffset` 和 strideOperand 沿与循环路径相同的 `arith.addi`、`arith.subi`、常量乘法和 index cast 规则构造成 `StrideExpr`;无法继续分解的 SSA 值作为叶子保留。 对同一 op 类型、同一 `rootBase` 的相邻候选 `i-1` 和 `i`,定义: @@ -362,7 +381,16 @@ step = combineStride(deltaBase, deltaStride, elemBytes, unitBytes) `step` 始终以该 op 的 strideOperand 单位表示。单位换算和精确缩放直接复用 4.2.1 与 4.2.5 的规则,不能证明精确换算时放弃该相邻关系。 -为比较两个非常量 `step` 是否相同,将 `StrideExpr` 按加减、常量乘法进行展平和常量折叠,得到规范化仿射形式“常量 + Σ(系数 × SSA 叶子)”。相同叶子按 Value 同一性合并,系数为零的项删除。两个 step 的规范化形式完全相同,才视为固定公差;不支持的非仿射运算保守地作为叶子,仅在复用同一 SSA 值时才能匹配。 +为比较两个非常量 `step` 是否相同,将 `StrideExpr` 按加减、常量乘法进行展平和常量折叠,得到规范化仿射形式“常量 + Σ(系数 × SSA 叶子)”。这里的“仿射”是相对于 SSA 叶子而言,并不要求叶子本身是原始输入的仿射函数。相同叶子按 Value 同一性合并,系数为零的项删除。两个 step 的规范化形式完全相同,才视为固定公差;不支持的非仿射运算保守地作为叶子,仅在复用同一 SSA 值时才能匹配。两个独立计算但语义上可能相等的非仿射结果是不同 Value,pass 不尝试证明它们等价。 + +例如 `%s` 由 `arith.select` 产生,虽然该运算不属于上述仿射语法,但下面三个 offset 的相邻 step 都可规范化为同一个 opaque leaf `%s`,因此仍能形成 `SequentialRun`: + +```mlir +%s = arith.select %cond, %lhs, %rhs : index +// offset: 0, %s, %s + %s +``` + +若相邻 step 分别依赖两个独立的非仿射结果 `%s0` 与 `%s1`,即使两者运行时可能相等,也不会匹配,除非此前的规范化或 CSE 已使它们成为同一个 SSA Value。 该分析可识别 base 与 offset 相互补偿的模式。例如 Element 类指令的三个候选满足: @@ -374,17 +402,20 @@ offset: -x, 48 - x, 96 - x 虽然 base 和 offset 都不是常量,但相邻地址的规范化 `step` 恒为 64。 -#### 4.3.2 分桶与 maximal runs +#### 4.3.2 分桶与 `SequentialRun` 在一个 block 内,将仍未改写的候选按 `(op 类型, rootBase)` 放入有序桶。不同桶的 op 可以在物理程序序中任意交错;每个桶只保留本桶候选的原始程序序。 -每个桶使用一次确定性的线性扫描,产生互不重叠的 maximal arithmetic runs: +`SequentialRun` 对应代码中的同名结构。无法再向后加入同 `step` 候选的 `SequentialRun` 称为最大 `SequentialRun`。 + +每个桶使用一次确定性的线性扫描,产生互不重叠的最大 `SequentialRun`: -1. 以当前候选作为 run 起点,下一候选与它形成第一个非零、合法的 `step`。 -2. 后续候选与前一候选的 `step` 若和当前 run 的规范化 step 相同,则加入 run。 -3. 若 step 不同或无法分析,则当前 run 立即结束;破坏公差的候选作为下一个 run 的起点。 +1. 以当前候选作为 `SequentialRun` 起点,下一候选与它形成第一个非零、合法的 `step`。 +2. 后续候选与前一候选的 `step` 若和当前 `SequentialRun` 的规范化 step 相同,则加入该 `SequentialRun`。 +3. 若 step 不同或无法分析,则当前候选 `SequentialRun` 立即结束;破坏公差的候选不加入该 `SequentialRun`。 4. 尚未形成合法 step 的两个候选若无法配对,丢弃前一个,以后一个重新尝试。 -5. run 长度达到 2 时即可确定 `step`,但长度至少为 3 才改写。已经归入已完成 run 的候选不再参与后续 run,因此不存在重叠、回溯或最长子序列冲突。 +5. `SequentialRun` 长度达到 2 时即可确定 `step`,但长度至少为 3 且通过 4.3.3 的合法性检查才接受并改写。接受后,破坏公差的候选作为下一个 `SequentialRun` 的起点;拒绝后,从该候选 `SequentialRun` 的最后一个候选(`end - 1`)重新尝试,使其可与破坏公差的候选组成新 `SequentialRun`。 +6. 只有已接受 `SequentialRun` 的候选会被消费,且不再参与后续 `SequentialRun`,因此最终接受的 `SequentialRun` 互不重叠。被拒 `SequentialRun` 最多复用一个尾候选,不进行内部回溯或最长子序列搜索。 该规则不追求全局最多命中,而是保证结果简单、线性、确定且符合程序序。 @@ -394,7 +425,7 @@ offset: -x, 48 - x, 96 - x pto.vsts %x, %other[%c0], %mask : ... %v1 = pto.vlds %base1[%off1] : ... // 本桶地址 64 %v2 = pto.vlds %base2[%off2] : ... // 本桶地址 128 -// 本桶形成一个 step = 64 的 maximal run +// 本桶形成一个 step = 64 的最大 SequentialRun ``` > **与循环路径的关键区别(链式 vs 共享)。** 循环路径同组 op 共享一个 `iter_arg`、全用 pre-update 指针、**不**链式传递(4.2.7,同迭代内同地址)。顺序路径相反,**必须**链式传递(前一条的 `updated_base` 喂后一条,见 4.3.4)。因此两条交错的序列(vlds 链 + vsts 链)是两条**独立**的链,各自穿针,不能合并;按 `(op 类型, rootBase)` 分桶使每条链自然独立。 @@ -404,21 +435,21 @@ pto.vsts %x, %other[%c0], %mask : ... 1. **op 尚未处于 Post-Update 形式。** 循环路径已经改写的 op 自动排除。 2. **地址可归一到同一 rootBase。** 只穿过 `pto.addptr`;其他指针变换不猜测别名关系。 3. **step 可精确换算且非零。** 复用 4.2.5 的单位、常量折叠和目标类型可表示性检查。 -4. **step 在 run 头可用。** 复用 4.2.5 的叶子可用性检查;定义在 run 头之后的 pure 仿射定义链可克隆到 run 头之前,含副作用或无法支配的定义使该 run 放弃。 -5. **整条 run 先分析、后物化。** 所有候选通过检查后才创建常量、`pto.addptr` 或新 op,拒绝 run 不留下残余 IR。 +4. **step 在 `SequentialRun` 头部可用。** 复用 4.2.5 的叶子可用性检查;定义在 `SequentialRun` 头部之后的 pure 定义链(包括产生 opaque leaf 的非仿射运算)可克隆到头部之前,含副作用、不可安全提前或无法支配的定义使该 `SequentialRun` 放弃。 +5. **整条 `SequentialRun` 先分析、后物化。** 所有候选通过检查后才创建常量、`pto.addptr` 或新 op,拒绝 `SequentialRun` 不留下残余 IR。 -顺序路径不重排 op,也不改变访问地址和访问顺序,因此不同候选之间的内存读写不会截断 run,无需额外别名分析。 +顺序路径不重排 op,也不改变访问地址和访问顺序,因此不同候选之间的内存读写不会截断 `SequentialRun`,无需额外别名分析。 #### 4.3.4 改写 -对每个 run,`stride_new` 就是 4.3.1 得到的规范化 `step`。初始指针直接从 run 首条 op 的实际操作数计算: +对每个 `SequentialRun`,`stride_new` 就是 4.3.1 得到的规范化 `step`。初始指针直接从 `SequentialRun` 首条 op 的实际操作数计算: ``` init_ptr = pto.addptr(first.base, (unitBytes/elemBytes)·first.strideOperand) ``` -这里复用循环路径的精确单位换算;不需要从 `rootBase` 重新构造绝对地址。将 run 中每条 op 替换为 post-update 形式,前一条的 `updated_base` 作为后一条的 base,strideOperand 替换为 `stride_new`: +这里复用循环路径的精确单位换算;不需要从 `rootBase` 重新构造绝对地址。将 `SequentialRun` 中每条 op 替换为 post-update 形式,前一条的 `updated_base` 作为后一条的 base,strideOperand 替换为 `stride_new`: ```mlir // vlds 变换后 @@ -427,7 +458,7 @@ init_ptr = pto.addptr(first.base, %v2, %ptr3 = pto.vlds %ptr2[%c64] : ... -> ..., !pto.ptr ``` -所有 run 在只读分析阶段确定后,先物化各自的 `stride_new` 和 `init_ptr`,再按原程序序替换候选 op,避免 erase op 使其他 run 保存的表达式失效。不同桶各自维护独立指针链,即使其 op 在原 block 中交错也互不影响。 +所有 `SequentialRun` 在只读分析阶段确定后,先物化各自的 `stride_new` 和 `init_ptr`,再按原程序序替换候选 op,避免 erase op 使其他 `SequentialRun` 保存的表达式失效。不同桶各自维护独立指针链,即使其 op 在原 block 中交错也互不影响。 ## 5. Pass 集成 @@ -486,15 +517,15 @@ def VPTOSoftPostUpdate : Pass<"vpto-soft-postupdate", "ModuleOp"> { 14. ~~实现物化阶段:类型一致性检查、叶子可用性检查(必要时克隆 pure 定义链)、常量按 `(值, 类型)` 复用。~~ 15. ~~补充回归测试:增量定义在消费者之后、多增量组合、weight=32 非常量增量的负向用例。~~ -### Step 3:顺序路径 +### ~~Step 3:顺序路径~~(已完成) -16. 抽取循环与顺序路径共享的 helper:精确 elemBytes/unitBytes 换算、`combineStride`、初始指针计算和 post-update op 泛型构造。 -17. 实现 base 归一化:沿 `pto.addptr` 得到 `(rootBase, baseOffset)`;复用 `StrideExpr` 构造规则,并增加仿射规范化、相等比较和按系数精确缩放,使两条路径都支持 `8*k` 一类可证明整除的非常量表达式。 -18. 调整驱动为两阶段:先内到外完成全部循环路径,再重新收集所有 block(包括 `scf.for` body)和剩余非 Post-Update op。 -19. 实现有序分桶与线性 maximal-run 检测:按 `(op 类型, rootBase)` 分桶,允许不同桶物理交错,run 互不重叠且不回溯。 -20. 复用循环路径的类型、精确换算、可用性和 pure 克隆检查,在整条 run 通过后物化 step 并链式改写。 -21. 更新 `Passes.td` 中 pass 描述,使其同时覆盖循环递推和 block 内顺序访问。 -22. 添加 `test/lit/vpto` 回归测试:固定 base 常量序列、变化 base 与 offset 抵消、非常量仿射 step、Block 类 `8*k` 精确缩放、for-body 循环路径未命中后转顺序路径、不同桶交错、多 maximal runs、公差破坏、零步长及无法精确换算的负向用例。 +16. ~~抽取循环与顺序路径共享的 helper:精确 elemBytes/unitBytes 换算、`combineStride`、初始指针计算和 post-update op 泛型构造。~~ +17. ~~实现 base 归一化:沿 `pto.addptr` 得到 `(rootBase, baseOffset)`;复用 `StrideExpr` 构造规则,并增加仿射规范化、相等比较和按系数精确缩放,使两条路径都支持 `8*k` 一类可证明整除的非常量表达式。~~ +18. ~~调整驱动为两阶段:先内到外完成全部循环路径,再重新收集所有 block(包括 `scf.for` body)和剩余非 Post-Update op。~~ +19. ~~实现有序分桶与线性 `SequentialRun` 检测:按 `(op 类型, rootBase)` 分桶,允许不同桶物理交错;接受的 `SequentialRun` 互不重叠,被拒 `SequentialRun` 从 `end - 1` 保留一个尾候选重试。~~ +20. ~~复用循环路径的类型、精确换算、可用性和 pure 克隆检查,在整条 `SequentialRun` 通过后物化 step 并链式改写。~~ +21. ~~更新 `Passes.td` 中 pass 描述,使其同时覆盖循环递推和 block 内顺序访问。~~ +22. ~~添加 `test/lit/vpto` 回归测试:固定 base 常量序列、变化 base 与 offset 抵消、非常量仿射 step、Block 类 `8*k` 精确缩放、for-body 循环路径未命中后转顺序路径、不同桶交错、多个最大 `SequentialRun`、公差破坏、零步长及无法精确换算的负向用例。~~ ### Step 4:扩展指令覆盖 diff --git a/lib/PTO/Transforms/VPTOSoftPostUpdate.cpp b/lib/PTO/Transforms/VPTOSoftPostUpdate.cpp index a9afca0d7..698ffa382 100644 --- a/lib/PTO/Transforms/VPTOSoftPostUpdate.cpp +++ b/lib/PTO/Transforms/VPTOSoftPostUpdate.cpp @@ -1610,12 +1610,16 @@ static void processSequentialBlock(Block *block, DominanceInfo &dominance, run.stepForm = firstStep->form; for (size_t i = start; i < end; ++i) run.candidates.push_back(&candidates[i]); - if (validateSequentialRun(run, dominance)) + if (validateSequentialRun(run, dominance)) { runs.push_back(std::move(run)); - - // Runs are deliberately non-overlapping. The candidate that broke the - // current stride becomes the start of the next run. - start = end; + // Accepted runs are deliberately non-overlapping. The candidate that + // broke the current stride becomes the start of the next run. + start = end; + } else { + // Reuse the rejected run's last candidate as the next head so it can + // form a new run with the candidate that broke the current stride. + start = end - 1; + } } } diff --git a/test/lit/vpto/soft_postupdate_sequential-run-restart.pto b/test/lit/vpto/soft_postupdate_sequential-run-restart.pto index bbcd5865b..a0b0eae5b 100644 --- a/test/lit/vpto/soft_postupdate_sequential-run-restart.pto +++ b/test/lit/vpto/soft_postupdate_sequential-run-restart.pto @@ -26,6 +26,15 @@ // CHECK: %{{.*}}, [[S2:%[a-zA-Z0-9_]+]] = pto.vlds [[S1]][%c32] // CHECK: %{{.*}}, %{{.*}} = pto.vlds [[S2]][%c32] +// If a two-candidate tentative run is rejected, its second candidate must be +// retried as the head of an overlapping later run. +// CHECK-LABEL: func.func @restart_after_overlapping_short_run +// CHECK: {{^ *}}%{{[^, ]+}} = pto.vlds {{%.*}}[%c0] +// CHECK: [[INIT:%[a-zA-Z0-9_]+]] = pto.addptr {{%.*}}, %c64 +// CHECK: %{{.*}}, [[O1:%[a-zA-Z0-9_]+]] = pto.vlds [[INIT]][%c136] +// CHECK: %{{.*}}, [[O2:%[a-zA-Z0-9_]+]] = pto.vlds [[O1]][%c136] +// CHECK: %{{.*}}, %{{.*}} = pto.vlds [[O2]][%c136] + module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind} { func.func @restart_after_zero_pair(%src: !pto.ptr, %sink: !pto.ptr) attributes {pto.kernel} { %c0 = arith.constant 0 : index @@ -67,4 +76,23 @@ module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind, %sink: !pto.ptr) attributes {pto.kernel} { + %c0 = arith.constant 0 : index + %c64 = arith.constant 64 : index + %c200 = arith.constant 200 : index + %c336 = arith.constant 336 : index + pto.vecscope { + %mask = pto.pset_b32 "PAT_ALL" : !pto.mask + %v0 = pto.vlds %src[%c0] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v0, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v1 = pto.vlds %src[%c64] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v1, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v2 = pto.vlds %src[%c200] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v2, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + %v3 = pto.vlds %src[%c336] : !pto.ptr -> !pto.vreg<64xf32> + pto.vsts %v3, %sink[%c0], %mask : !pto.vreg<64xf32>, !pto.ptr, !pto.mask + } + return + } }