missing VMI features for high-perf simd kernels

docs/feature-gaps/vmi/01_scalar_gm_dcache_bypass/README.md

@@ -0,0 +1,41 @@
+# 01 — Scalar GM dcache-bypass store/load
+
+## Algorithm need
+
+Per-block quantization over **32×32** groups produces one f32 scale factor (SF)
+per group. That scalar must land in GM (and later be re-read). Today the tiny
+write still goes through bulk MTE, so setup/alignment/sync dominate and keep
+VMI short of ASC bandwidth on the SF path.
+
+## Current slow path
+
+Stage the SF in UB, then `pto.copy_ubuf_to_gm` with a padded burst (here 32B
+for a 4B payload). Checked-in dump: `lowered_vpto.pto` — key op is
+`pto.copy_ubuf_to_gm` (no scalar GM path).
+
+Reproduce:
+
+```bash
+PTO_TEST_OPT=.../pto-test-opt
+PASS='-vmi-lower-unified-to-legacy -vmi-mask-granularity-assignment -vmi-layout-assignment -vmi-to-vpto'
+$PTO_TEST_OPT current_slow_vmi.pto $PASS -o lowered_vpto.pto
+```
+
+## Desired VMI + current failure
+
+Direct `pto.vmi.vstore` of a slots=1 vector to `!pto.ptr<f32, gm>` with
+`dcache_bypass` (see `desired_vmi.pto` / `desired_vmi.py`):
+
+```
+error: 'pto.vmi.group_store' op requires memory destination to be UB-backed
+```
+
+## Target MI + bisheng status
+
+`target_mi.pto` uses `pto.store_scalar ... {dcache_bypass}` on GM.
+
+| Step | Result |
+|---|---|
+| `ptoas ... --emit-vpto` | OK |
+| `ptoas ... --emit-vpto-llvm-ir` | OK |
+| `bisheng ... -c -x ir` → `.o` | OK |

docs/feature-gaps/vmi/01_scalar_gm_dcache_bypass/current_slow_vmi.pto

@@ -0,0 +1,24 @@
+// Copyright (c) 2026 Huawei Technologies Co., Ltd.
+// This program is free software, you can redistribute it and/or modify it under the terms and conditions of
+// CANN Open Software License Agreement Version 2.0 (the "License").
+// Please refer to the License for details. You may not use this file except in compliance with the License.
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
+// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
+// See LICENSE in the root of the software repository for the full text of the License.
+
+// Current slow path: SF already staged in UB; bulk MTE to GM for a tiny payload.
+
+module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind<vector>} {
+  func.func @scale_factor_bulk_mte(
+      %sf_ub: !pto.ptr<f32, ub>,
+      %sf_gm: !pto.ptr<f32, gm>) {
+    %c0_i64 = arith.constant 0 : i64
+    %c1_i64 = arith.constant 1 : i64
+    %c32_i64 = arith.constant 32 : i64
+    // 4B useful SF / 32B burst — MTE setup dominates.
+    pto.copy_ubuf_to_gm %sf_ub, %sf_gm,
+        %c0_i64, %c1_i64, %c32_i64, %c0_i64, %c32_i64, %c32_i64
+        : !pto.ptr<f32, ub>, !pto.ptr<f32, gm>, i64, i64, i64, i64, i64, i64
+    return
+  }
+}

docs/feature-gaps/vmi/01_scalar_gm_dcache_bypass/current_slow_vmi.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2026 Huawei Technologies Co., Ltd.
+# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
+# CANN Open Software License Agreement Version 2.0 (the "License").
+# Please refer to the License for details. You may not use this file except in compliance with the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE in the root of the software repository for the full text of the License.
+
+"""Python DSL form of ``current_slow_vmi.pto`` — bulk UB→GM MTE for a tiny SF.
+
+Canonical IR: ``current_slow_vmi.pto``. Emit: ``scale_factor_bulk_mte.compile().mlir_text()``.
+"""
+
+from ptodsl import pto
+
+
+@pto.jit(target="a5", backend="vpto", mode="explicit")
+def scale_factor_bulk_mte(sf_gm: pto.ptr(pto.f32, "gm")):
+    # SF staged in UB; 4B useful / 32B burst — MTE setup dominates.
+    sf_ub = pto.alloc_tile(shape=[1, 8], dtype=pto.f32)
+    pto.mte_ub_gm(sf_ub.as_ptr(), sf_gm, 32, nburst=(1, 32, 32))
+
+
+if __name__ == "__main__":
+    print(scale_factor_bulk_mte.compile().mlir_text())

docs/feature-gaps/vmi/01_scalar_gm_dcache_bypass/desired_vmi.pto

@@ -0,0 +1,22 @@
+// Copyright (c) 2026 Huawei Technologies Co., Ltd.
+// This program is free software, you can redistribute it and/or modify it under the terms and conditions of
+// CANN Open Software License Agreement Version 2.0 (the "License").
+// Please refer to the License for details. You may not use this file except in compliance with the License.
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
+// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
+// See LICENSE in the root of the software repository for the full text of the License.
+
+// Idiomatic VMI: direct scalar GM store with dcache bypass (does not lower today).
+
+module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind<vector>} {
+  func.func @scale_factor_gm_bypass(
+      %sf_gm: !pto.ptr<f32, gm>,
+      %scale: !pto.vmi.vreg<1xf32, #pto.vmi.layout<num_groups = 1, slots = 1>>,
+      %off: index) {
+    %c1 = arith.constant 1 : index
+    pto.vmi.vstore %scale, %sf_gm[%off], %c1 {group = 1, dcache_bypass}
+        : !pto.vmi.vreg<1xf32, #pto.vmi.layout<num_groups = 1, slots = 1>>,
+          !pto.ptr<f32, gm>
+    return
+  }
+}

docs/feature-gaps/vmi/01_scalar_gm_dcache_bypass/desired_vmi.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2026 Huawei Technologies Co., Ltd.
+# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
+# CANN Open Software License Agreement Version 2.0 (the "License").
+# Please refer to the License for details. You may not use this file except in compliance with the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE in the root of the software repository for the full text of the License.
+
+"""Python DSL form of ``desired_vmi.pto`` — scalar GM store with dcache bypass.
+
+Canonical IR: ``desired_vmi.pto`` (``{group = 1, dcache_bypass}``).
+PTODSL ``vstore`` does not yet expose ``dcache_bypass``; IR remains authoritative.
+"""
+
+from ptodsl import pto
+
+
+@pto.jit(target="a5", backend="vpto", mode="explicit")
+def scale_factor_gm_bypass(sf_gm: pto.ptr(pto.f32, "gm")):
+    ub = pto.alloc_tile(shape=[1, 8], dtype=pto.f32)
+    off = pto.const(0, dtype=pto.index)
+    scale = pto.vmi.vload(ub.as_ptr(), off, size=1)
+    # Desired IR also sets dcache_bypass on this store.
+    pto.vmi.vstore(
+        scale,
+        sf_gm,
+        off,
+        group=1,
+        stride=pto.const(1, dtype=pto.index),
+    )
+
+
+if __name__ == "__main__":
+    print(scale_factor_gm_bypass.compile().mlir_text())

docs/feature-gaps/vmi/01_scalar_gm_dcache_bypass/lowered_vpto.pto

@@ -0,0 +1,10 @@
+module attributes {pto.kernel_kind = #pto.kernel_kind<vector>, pto.target_arch = "a5"} {
+  func.func @scale_factor_bulk_mte(%arg0: !pto.ptr<f32, ub>, %arg1: !pto.ptr<f32, gm>) {
+    %c0_i64 = arith.constant 0 : i64
+    %c1_i64 = arith.constant 1 : i64
+    %c32_i64 = arith.constant 32 : i64
+    pto.copy_ubuf_to_gm %arg0, %arg1, %c0_i64, %c1_i64, %c32_i64, %c0_i64, %c32_i64, %c32_i64 : !pto.ptr<f32, ub>, !pto.ptr<f32, gm>, i64, i64, i64, i64, i64, i64
+    return
+  }
+}
+

docs/feature-gaps/vmi/01_scalar_gm_dcache_bypass/target_mi.pto

@@ -0,0 +1,20 @@
+// Copyright (c) 2026 Huawei Technologies Co., Ltd.
+// This program is free software, you can redistribute it and/or modify it under the terms and conditions of
+// CANN Open Software License Agreement Version 2.0 (the "License").
+// Please refer to the License for details. You may not use this file except in compliance with the License.
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
+// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
+// See LICENSE in the root of the software repository for the full text of the License.
+
+// Desired MI: scalar GM store with dcache bypass (no bulk MTE).
+
+module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind<vector>} {
+  func.func @scale_factor_gm_bypass_mi(
+      %sf_gm: !pto.ptr<f32, gm>,
+      %scale: f32) {
+    %c0 = arith.constant 0 : index
+    pto.store_scalar %scale, %sf_gm[%c0] {dcache_bypass}
+        : !pto.ptr<f32, gm>, f32
+    return
+  }
+}

docs/feature-gaps/vmi/02_compact_inv_vbrc/README.md

@@ -0,0 +1,40 @@
+# 02 — Compact inverse `vbrc` (no pad + BRC)
+
+## Algorithm need
+
+Quant kernels compute per-group reciprocal scales, then broadcast each inverse
+across the group's lanes before the multiply. A **compact** inverse (one slot
+per group, no pad) fan-out is what ASC-level BW wants; padding + BRC reload
+adds UB traffic and ops on the hot path.
+
+## Current slow path
+
+Materialize recip scales as `#pto.vmi.layout<num_groups=8, slots=8>`,
+`group_store` them, then reload with `dist_mode = "brc"`. Checked-in dump:
+`lowered_vpto.pto` — key ops `pto.vsts` with `PAT_VL8`, then
+`pto.vlds {dist = "BRC_B32"}`.
+
+Reproduce:
+
+```bash
+$PTO_TEST_OPT current_slow_vmi.pto $PASS -o lowered_vpto.pto
+```
+
+## Desired VMI + current failure
+
+Compact `vbrc` from `num_groups=8, slots=1` (see `desired_vmi.pto` / `desired_vmi.py`):
+
+```
+error: VMI-LAYOUT-CONTRACT: pto.vmi.group_broadcast has no registered layout support:
+operand#0=...num_groups = 8, slots = 1... result#0=...contiguous...
+```
+
+## Target MI + bisheng status
+
+`target_mi.pto` is a `BRC_B32` load (plus a consumer store inside `vecscope`).
+
+| Step | Result |
+|---|---|
+| `ptoas ... --emit-vpto` | OK |
+| `ptoas ... --emit-vpto-llvm-ir` | OK |
+| `bisheng ... -c -x ir` → `.o` | OK |

docs/feature-gaps/vmi/02_compact_inv_vbrc/current_slow_vmi.pto

@@ -0,0 +1,25 @@
+// Copyright (c) 2026 Huawei Technologies Co., Ltd.
+// This program is free software, you can redistribute it and/or modify it under the terms and conditions of
+// CANN Open Software License Agreement Version 2.0 (the "License").
+// Please refer to the License for details. You may not use this file except in compliance with the License.
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
+// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
+// See LICENSE in the root of the software repository for the full text of the License.
+
+// Current slow path: pad recip scales to slots=8, store, then BRC reload.
+
+module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind<vector>} {
+  func.func @padded_inv_brc(
+      %inv_ub: !pto.ptr<f32, ub>,
+      %inv_padded: !pto.vmi.vreg<8xf32, #pto.vmi.layout<num_groups = 8, slots = 8>>)
+      -> !pto.vmi.vreg<64xf32> {
+    %c0 = arith.constant 0 : index
+    %c1 = arith.constant 1 : index
+    pto.vmi.vstore %inv_padded, %inv_ub[%c0], %c1 {group = 8}
+        : !pto.vmi.vreg<8xf32, #pto.vmi.layout<num_groups = 8, slots = 8>>,
+          !pto.ptr<f32, ub>
+    %inv_brc = pto.vmi.vload %inv_ub[%c0] {dist_mode = "brc"}
+        : !pto.ptr<f32, ub> -> !pto.vmi.vreg<64xf32>
+    return %inv_brc : !pto.vmi.vreg<64xf32>
+  }
+}

docs/feature-gaps/vmi/02_compact_inv_vbrc/current_slow_vmi.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2026 Huawei Technologies Co., Ltd.
+# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
+# CANN Open Software License Agreement Version 2.0 (the "License").
+# Please refer to the License for details. You may not use this file except in compliance with the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE in the root of the software repository for the full text of the License.
+
+"""Python DSL form of ``current_slow_vmi.pto`` — pad recip to slots=8, store, BRC reload.
+
+Canonical IR: ``current_slow_vmi.pto``.
+"""
+
+from ptodsl import pto
+
+
+@pto.jit(target="a5", backend="vpto", mode="explicit")
+def padded_inv_brc():
+    inv_ub = pto.alloc_tile(shape=[1, 8], dtype=pto.f32)
+    off = pto.const(0, dtype=pto.index)
+    inv_padded = pto.vmi.vload(inv_ub.as_ptr(), off, size=8)
+    pto.vmi.vstore(
+        inv_padded,
+        inv_ub.as_ptr(),
+        off,
+        group=8,
+        stride=pto.const(1, dtype=pto.index),
+    )
+    inv_brc = pto.vmi.vload(inv_ub.as_ptr(), off, size=64, dist_mode="brc")
+    _ = inv_brc
+
+
+if __name__ == "__main__":
+    print(padded_inv_brc.compile().mlir_text())

docs/feature-gaps/vmi/02_compact_inv_vbrc/desired_vmi.pto

@@ -0,0 +1,20 @@
+// Copyright (c) 2026 Huawei Technologies Co., Ltd.
+// This program is free software, you can redistribute it and/or modify it under the terms and conditions of
+// CANN Open Software License Agreement Version 2.0 (the "License").
+// Please refer to the License for details. You may not use this file except in compliance with the License.
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
+// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
+// See LICENSE in the root of the software repository for the full text of the License.
+
+// Idiomatic VMI: compact inverse vbrc from slots=1 (no pad round-trip).
+
+module attributes {pto.target_arch = "a5", pto.kernel_kind = #pto.kernel_kind<vector>} {
+  func.func @compact_inv_vbrc(
+      %inv_compact: !pto.vmi.vreg<8xf32, #pto.vmi.layout<num_groups = 8, slots = 1>>)
+      -> !pto.vmi.vreg<64xf32> {
+    %inv = pto.vmi.vbrc %inv_compact {group = 8}
+        : !pto.vmi.vreg<8xf32, #pto.vmi.layout<num_groups = 8, slots = 1>>
+        -> !pto.vmi.vreg<64xf32>
+    return %inv : !pto.vmi.vreg<64xf32>
+  }
+}

docs/feature-gaps/vmi/02_compact_inv_vbrc/desired_vmi.py

@@ -0,0 +1,27 @@
+# Copyright (c) 2026 Huawei Technologies Co., Ltd.
+# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
+# CANN Open Software License Agreement Version 2.0 (the "License").
+# Please refer to the License for details. You may not use this file except in compliance with the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE in the root of the software repository for the full text of the License.
+
+"""Python DSL form of ``desired_vmi.pto`` — compact inverse ``vbrc`` (no pad round-trip).
+
+Canonical IR: ``desired_vmi.pto``.
+"""
+
+from ptodsl import pto
+
+
+@pto.jit(target="a5", backend="vpto", mode="explicit")
+def compact_inv_vbrc():
+    inv_ub = pto.alloc_tile(shape=[1, 8], dtype=pto.f32)
+    off = pto.const(0, dtype=pto.index)
+    inv_compact = pto.vmi.vload(inv_ub.as_ptr(), off, size=8)
+    inv = pto.vmi.vbrc(inv_compact, size=64, group=8)
+    _ = inv
+
+
+if __name__ == "__main__":
+    print(compact_inv_vbrc.compile().mlir_text())

docs/feature-gaps/vmi/02_compact_inv_vbrc/lowered_vpto.pto

@@ -0,0 +1,12 @@
+module attributes {pto.kernel_kind = #pto.kernel_kind<vector>, pto.target_arch = "a5"} {
+  func.func @padded_inv_brc(%arg0: !pto.ptr<f32, ub>, %arg1: !pto.vreg<64xf32>) -> !pto.vreg<64xf32> {
+    %c0 = arith.constant 0 : index
+    %c1 = arith.constant 1 : index
+    %0 = pto.pge_b32 "PAT_VL8" : !pto.mask<b32>
+    pto.vsts %arg1, %arg0[%c0], %0 : !pto.vreg<64xf32>, !pto.ptr<f32, ub>, !pto.mask<b32>
+    %c0_0 = arith.constant 0 : index
+    %result = pto.vlds %arg0[%c0] {dist = "BRC_B32"} : !pto.ptr<f32, ub> -> !pto.vreg<64xf32>
+    return %result : !pto.vreg<64xf32>
+  }
+}
+

docs/feature-gaps/vmi/02_compact_inv_vbrc/target_mi.pto

@@ -0,0 +1,25 @@
+// Copyright (c) 2026 Huawei Technologies Co., Ltd.
+// This program is free software, you can redistribute it and/or modify it under the terms and conditions of
+// CANN Open Software License Agreement Version 2.0 (the "License").
+// Please refer to the License for details. You may not use this file except in compliance with the License.
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
+// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
+// See LICENSE in the root of the software repository for the full text of the License.
+
+// Desired MI: BRC_B32 from the compact inverse line (no pad store + reload).
+
+module {
+  func.func @compact_inv_vbrc_mi(
+      %inv_ub: !pto.ptr<f32, ub>,
+      %off: index,
+      %dst: !pto.ptr<f32, ub>) {
+    pto.vecscope {
+      %inv = pto.vlds %inv_ub[%off] {dist = "BRC_B32"}
+          : !pto.ptr<f32, ub> -> !pto.vreg<64xf32>
+      %m = pto.pset_b32 "PAT_ALL" : !pto.mask<b32>
+      pto.vsts %inv, %dst[%off], %m
+          : !pto.vreg<64xf32>, !pto.ptr<f32, ub>, !pto.mask<b32>
+    }
+    return
+  }
+}